diff --git a/BUILD b/BUILD index c0d79f2eae8..44a20c3e43f 100644 --- a/BUILD +++ b/BUILD @@ -1659,6 +1659,7 @@ grpc_cc_library( external_deps = [ "absl/base:core_headers", "absl/status", + "absl/strings:str_format", "absl/types:optional", "absl/types:variant", "absl/utility", @@ -2820,7 +2821,6 @@ grpc_cc_library( "absl/strings", "absl/strings:str_format", "absl/types:optional", - "absl/utility", ], deps = [ "event_engine_base_hdrs", @@ -3234,6 +3234,7 @@ grpc_cc_library( "src/core/lib/surface/call.cc", "src/core/lib/surface/call_details.cc", "src/core/lib/surface/call_log_batch.cc", + "src/core/lib/surface/call_trace.cc", "src/core/lib/surface/channel.cc", "src/core/lib/surface/channel_ping.cc", "src/core/lib/surface/completion_queue.cc", @@ -3244,6 +3245,7 @@ grpc_cc_library( "src/core/lib/surface/server.cc", "src/core/lib/surface/validate_metadata.cc", "src/core/lib/surface/version.cc", + "src/core/lib/transport/call_fragments.cc", "src/core/lib/transport/connectivity_state.cc", "src/core/lib/transport/error_utils.cc", "src/core/lib/transport/metadata_batch.cc", @@ -3324,6 +3326,7 @@ grpc_cc_library( "src/core/lib/surface/builtins.h", "src/core/lib/surface/call.h", "src/core/lib/surface/call_test_only.h", + "src/core/lib/surface/call_trace.h", "src/core/lib/surface/channel.h", "src/core/lib/surface/completion_queue.h", "src/core/lib/surface/completion_queue_factory.h", @@ -3334,6 +3337,7 @@ grpc_cc_library( "src/core/lib/surface/validate_metadata.h", "src/core/lib/transport/connectivity_state.h", "src/core/lib/transport/metadata_batch.h", + "src/core/lib/transport/call_fragments.h", "src/core/lib/transport/parsed_metadata.h", "src/core/lib/transport/status_conversion.h", "src/core/lib/transport/timeout_encoding.h", @@ -3359,6 +3363,7 @@ grpc_cc_library( ], external_deps = [ "absl/base:core_headers", + "absl/cleanup", "absl/container:flat_hash_map", "absl/container:inlined_vector", "absl/functional:any_invocable", @@ -3417,11 +3422,13 @@ grpc_cc_library( "iomgr_timer", "json", "latch", + "match", "memory_quota", "no_destruct", "notification", "orphanable", "packed_table", + "pipe", "poll", "pollset_set", "promise", diff --git a/CMakeLists.txt b/CMakeLists.txt index d5baa956924..20049eb3eaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -865,6 +865,7 @@ if(gRPC_BUILD_TESTS) add_dependencies(buildtests_cxx byte_buffer_test) add_dependencies(buildtests_cxx c_slice_buffer_test) add_dependencies(buildtests_cxx call_finalization_test) + add_dependencies(buildtests_cxx call_fragments_test) add_dependencies(buildtests_cxx call_push_pull_test) add_dependencies(buildtests_cxx cancel_ares_query_test) add_dependencies(buildtests_cxx cel_authorization_engine_test) @@ -2309,6 +2310,7 @@ add_library(grpc src/core/lib/surface/call.cc src/core/lib/surface/call_details.cc src/core/lib/surface/call_log_batch.cc + src/core/lib/surface/call_trace.cc src/core/lib/surface/channel.cc src/core/lib/surface/channel_init.cc src/core/lib/surface/channel_ping.cc @@ -2324,6 +2326,7 @@ add_library(grpc src/core/lib/surface/validate_metadata.cc src/core/lib/surface/version.cc src/core/lib/transport/bdp_estimator.cc + src/core/lib/transport/call_fragments.cc src/core/lib/transport/connectivity_state.cc src/core/lib/transport/error_utils.cc src/core/lib/transport/handshaker.cc @@ -2407,6 +2410,7 @@ target_link_libraries(grpc ${_gRPC_RE2_LIBRARIES} ${_gRPC_UPB_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + absl::cleanup absl::flat_hash_map absl::flat_hash_set absl::inlined_vector @@ -2877,6 +2881,7 @@ add_library(grpc_unsecure src/core/lib/surface/call.cc src/core/lib/surface/call_details.cc src/core/lib/surface/call_log_batch.cc + src/core/lib/surface/call_trace.cc src/core/lib/surface/channel.cc src/core/lib/surface/channel_init.cc src/core/lib/surface/channel_ping.cc @@ -2892,6 +2897,7 @@ add_library(grpc_unsecure src/core/lib/surface/validate_metadata.cc src/core/lib/surface/version.cc src/core/lib/transport/bdp_estimator.cc + src/core/lib/transport/call_fragments.cc src/core/lib/transport/connectivity_state.cc src/core/lib/transport/error_utils.cc src/core/lib/transport/handshaker.cc @@ -2951,6 +2957,7 @@ target_link_libraries(grpc_unsecure ${_gRPC_RE2_LIBRARIES} ${_gRPC_UPB_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} + absl::cleanup absl::flat_hash_map absl::flat_hash_set absl::inlined_vector @@ -3207,7 +3214,6 @@ target_link_libraries(grpc++ ${_gRPC_BASELIB_LIBRARIES} ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::cleanup grpc ) @@ -6528,7 +6534,6 @@ target_include_directories(binder_transport_test target_link_libraries(binder_transport_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::cleanup grpc_test_util ) @@ -6721,6 +6726,41 @@ target_link_libraries(call_finalization_test ) +endif() +if(gRPC_BUILD_TESTS) + +add_executable(call_fragments_test + test/core/transport/call_fragments_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + +target_include_directories(call_fragments_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(call_fragments_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util +) + + endif() if(gRPC_BUILD_TESTS) @@ -9139,7 +9179,6 @@ target_include_directories(endpoint_binder_pool_test target_link_libraries(endpoint_binder_pool_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::cleanup grpc_test_util ) @@ -9774,7 +9813,6 @@ target_include_directories(fake_binder_test target_link_libraries(fake_binder_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::cleanup grpc_test_util ) @@ -17775,7 +17813,6 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) target_link_libraries(tcp_posix_socket_utils_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::cleanup grpc_test_util ) @@ -18027,7 +18064,6 @@ target_include_directories(test_core_gprpp_load_file_test target_link_libraries(test_core_gprpp_load_file_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::cleanup grpc_test_util ) @@ -19198,7 +19234,6 @@ target_include_directories(transport_stream_receiver_test target_link_libraries(transport_stream_receiver_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::cleanup grpc_test_util ) @@ -19745,7 +19780,6 @@ target_include_directories(wire_reader_test target_link_libraries(wire_reader_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::cleanup grpc_test_util ) @@ -19848,7 +19882,6 @@ target_include_directories(wire_writer_test target_link_libraries(wire_writer_test ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} - absl::cleanup grpc_test_util ) @@ -22378,7 +22411,7 @@ generate_pkgconfig( "gRPC" "high performance general RPC framework" "${gRPC_CORE_VERSION}" - "gpr openssl absl_any_invocable absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" + "gpr openssl absl_any_invocable absl_base absl_bind_front absl_cleanup absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" "-lgrpc -laddress_sorting -lre2 -lupb -lcares -lz" "" "grpc.pc") @@ -22388,7 +22421,7 @@ generate_pkgconfig( "gRPC unsecure" "high performance general RPC framework without SSL" "${gRPC_CORE_VERSION}" - "gpr absl_any_invocable absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" + "gpr absl_any_invocable absl_base absl_bind_front absl_cleanup absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" "-lgrpc_unsecure" "" "grpc_unsecure.pc") @@ -22408,7 +22441,7 @@ generate_pkgconfig( "gRPC++ unsecure" "C++ wrapper for gRPC without SSL" "${gRPC_CPP_VERSION}" - "grpc_unsecure absl_any_invocable absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" + "grpc_unsecure absl_any_invocable absl_base absl_bind_front absl_cleanup absl_cord absl_core_headers absl_flat_hash_map absl_flat_hash_set absl_function_ref absl_hash absl_inlined_vector absl_memory absl_optional absl_random_random absl_span absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_type_traits absl_utility absl_variant" "-lgrpc++_unsecure" "" "grpc++_unsecure.pc") diff --git a/Makefile b/Makefile index 0e14e11b83a..1ffdace833a 100644 --- a/Makefile +++ b/Makefile @@ -1586,6 +1586,7 @@ LIBGRPC_SRC = \ src/core/lib/surface/call.cc \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ + src/core/lib/surface/call_trace.cc \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel_init.cc \ src/core/lib/surface/channel_ping.cc \ @@ -1601,6 +1602,7 @@ LIBGRPC_SRC = \ src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ + src/core/lib/transport/call_fragments.cc \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/error_utils.cc \ src/core/lib/transport/handshaker.cc \ @@ -2018,6 +2020,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/surface/call.cc \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ + src/core/lib/surface/call_trace.cc \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel_init.cc \ src/core/lib/surface/channel_ping.cc \ @@ -2033,6 +2036,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ + src/core/lib/transport/call_fragments.cc \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/error_utils.cc \ src/core/lib/transport/handshaker.cc \ diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index cc979cd34f1..5d00a416642 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -26,6 +26,9 @@ EXPERIMENTS = { ], }, "off": { + "core_end2end_test": [ + "promise_based_client_call", + ], "endpoint_test": [ "tcp_frame_size_tuning", "tcp_rcv_lowat", @@ -44,6 +47,9 @@ EXPERIMENTS = { "hpack_test": [ "periodic_resource_quota_reclamation", ], + "lame_client_test": [ + "promise_based_client_call", + ], "promise_test": [ "periodic_resource_quota_reclamation", ], diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 748999dd5c9..7a9660e244e 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -866,6 +866,7 @@ libs: - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h + - src/core/lib/promise/pipe.h - src/core/lib/promise/poll.h - src/core/lib/promise/promise.h - src/core/lib/promise/race.h @@ -955,6 +956,7 @@ libs: - src/core/lib/surface/builtins.h - src/core/lib/surface/call.h - src/core/lib/surface/call_test_only.h + - src/core/lib/surface/call_trace.h - src/core/lib/surface/channel.h - src/core/lib/surface/channel_init.h - src/core/lib/surface/channel_stack_type.h @@ -967,6 +969,7 @@ libs: - src/core/lib/surface/server.h - src/core/lib/surface/validate_metadata.h - src/core/lib/transport/bdp_estimator.h + - src/core/lib/transport/call_fragments.h - src/core/lib/transport/connectivity_state.h - src/core/lib/transport/error_utils.h - src/core/lib/transport/handshaker.h @@ -1643,6 +1646,7 @@ libs: - src/core/lib/surface/call.cc - src/core/lib/surface/call_details.cc - src/core/lib/surface/call_log_batch.cc + - src/core/lib/surface/call_trace.cc - src/core/lib/surface/channel.cc - src/core/lib/surface/channel_init.cc - src/core/lib/surface/channel_ping.cc @@ -1658,6 +1662,7 @@ libs: - src/core/lib/surface/validate_metadata.cc - src/core/lib/surface/version.cc - src/core/lib/transport/bdp_estimator.cc + - src/core/lib/transport/call_fragments.cc - src/core/lib/transport/connectivity_state.cc - src/core/lib/transport/error_utils.cc - src/core/lib/transport/handshaker.cc @@ -1703,6 +1708,7 @@ libs: - src/core/tsi/transport_security.cc - src/core/tsi/transport_security_grpc.cc deps: + - absl/cleanup:cleanup - absl/container:flat_hash_map - absl/container:flat_hash_set - absl/container:inlined_vector @@ -2064,6 +2070,7 @@ libs: - src/core/lib/promise/latch.h - src/core/lib/promise/loop.h - src/core/lib/promise/map.h + - src/core/lib/promise/pipe.h - src/core/lib/promise/poll.h - src/core/lib/promise/promise.h - src/core/lib/promise/race.h @@ -2122,6 +2129,7 @@ libs: - src/core/lib/surface/builtins.h - src/core/lib/surface/call.h - src/core/lib/surface/call_test_only.h + - src/core/lib/surface/call_trace.h - src/core/lib/surface/channel.h - src/core/lib/surface/channel_init.h - src/core/lib/surface/channel_stack_type.h @@ -2134,6 +2142,7 @@ libs: - src/core/lib/surface/server.h - src/core/lib/surface/validate_metadata.h - src/core/lib/transport/bdp_estimator.h + - src/core/lib/transport/call_fragments.h - src/core/lib/transport/connectivity_state.h - src/core/lib/transport/error_utils.h - src/core/lib/transport/handshaker.h @@ -2444,6 +2453,7 @@ libs: - src/core/lib/surface/call.cc - src/core/lib/surface/call_details.cc - src/core/lib/surface/call_log_batch.cc + - src/core/lib/surface/call_trace.cc - src/core/lib/surface/channel.cc - src/core/lib/surface/channel_init.cc - src/core/lib/surface/channel_ping.cc @@ -2459,6 +2469,7 @@ libs: - src/core/lib/surface/validate_metadata.cc - src/core/lib/surface/version.cc - src/core/lib/transport/bdp_estimator.cc + - src/core/lib/transport/call_fragments.cc - src/core/lib/transport/connectivity_state.cc - src/core/lib/transport/error_utils.cc - src/core/lib/transport/handshaker.cc @@ -2480,6 +2491,7 @@ libs: - src/core/tsi/transport_security.cc - src/core/tsi/transport_security_grpc.cc deps: + - absl/cleanup:cleanup - absl/container:flat_hash_map - absl/container:flat_hash_set - absl/container:inlined_vector @@ -2839,7 +2851,6 @@ libs: - src/cpp/util/string_ref.cc - src/cpp/util/time_cc.cc deps: - - absl/cleanup:cleanup - grpc baselib: true - name: grpc++_alts @@ -4383,7 +4394,6 @@ targets: - test/core/transport/binder/binder_transport_test.cc - test/core/transport/binder/mock_objects.cc deps: - - absl/cleanup:cleanup - grpc_test_util uses_polling: false - name: bitset_test @@ -4463,6 +4473,16 @@ targets: - test/core/channel/call_finalization_test.cc deps: - grpc_test_util +- name: call_fragments_test + gtest: true + build: test + language: c++ + headers: + - test/core/promise/test_context.h + src: + - test/core/transport/call_fragments_test.cc + deps: + - grpc_test_util - name: call_push_pull_test gtest: true build: test @@ -5549,7 +5569,6 @@ targets: - test/core/transport/binder/endpoint_binder_pool_test.cc - test/core/transport/binder/mock_objects.cc deps: - - absl/cleanup:cleanup - grpc_test_util uses_polling: false - name: endpoint_config_test @@ -5965,7 +5984,6 @@ targets: - test/core/transport/binder/end2end/fake_binder.cc - test/core/transport/binder/end2end/fake_binder_test.cc deps: - - absl/cleanup:cleanup - grpc_test_util uses_polling: false - name: fake_resolver_test @@ -9721,7 +9739,6 @@ targets: - src/core/lib/iomgr/socket_mutator.cc - test/core/event_engine/posix/tcp_posix_socket_utils_test.cc deps: - - absl/cleanup:cleanup - grpc_test_util platforms: - linux @@ -9857,7 +9874,6 @@ targets: - src/core/lib/gprpp/load_file.cc - test/core/gprpp/load_file_test.cc deps: - - absl/cleanup:cleanup - grpc_test_util uses_polling: false - name: test_core_gprpp_time_test @@ -10408,7 +10424,6 @@ targets: - src/cpp/util/time_cc.cc - test/core/transport/binder/transport_stream_receiver_test.cc deps: - - absl/cleanup:cleanup - grpc_test_util uses_polling: false - name: try_join_test @@ -10687,7 +10702,6 @@ targets: - test/core/transport/binder/mock_objects.cc - test/core/transport/binder/wire_reader_test.cc deps: - - absl/cleanup:cleanup - grpc_test_util uses_polling: false - name: wire_writer_test @@ -10796,7 +10810,6 @@ targets: - test/core/transport/binder/mock_objects.cc - test/core/transport/binder/wire_writer_test.cc deps: - - absl/cleanup:cleanup - grpc_test_util uses_polling: false - name: work_queue_test diff --git a/config.m4 b/config.m4 index 5a22e459b13..9f8a6093fe2 100644 --- a/config.m4 +++ b/config.m4 @@ -710,6 +710,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/surface/call.cc \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ + src/core/lib/surface/call_trace.cc \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel_init.cc \ src/core/lib/surface/channel_ping.cc \ @@ -725,6 +726,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/surface/validate_metadata.cc \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ + src/core/lib/transport/call_fragments.cc \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/error_utils.cc \ src/core/lib/transport/handshaker.cc \ diff --git a/config.w32 b/config.w32 index 40225dbaf8e..77e6d65b751 100644 --- a/config.w32 +++ b/config.w32 @@ -676,6 +676,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\surface\\call.cc " + "src\\core\\lib\\surface\\call_details.cc " + "src\\core\\lib\\surface\\call_log_batch.cc " + + "src\\core\\lib\\surface\\call_trace.cc " + "src\\core\\lib\\surface\\channel.cc " + "src\\core\\lib\\surface\\channel_init.cc " + "src\\core\\lib\\surface\\channel_ping.cc " + @@ -691,6 +692,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\surface\\validate_metadata.cc " + "src\\core\\lib\\surface\\version.cc " + "src\\core\\lib\\transport\\bdp_estimator.cc " + + "src\\core\\lib\\transport\\call_fragments.cc " + "src\\core\\lib\\transport\\connectivity_state.cc " + "src\\core\\lib\\transport\\error_utils.cc " + "src\\core\\lib\\transport\\handshaker.cc " + diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 8b11c408367..eea00352104 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -840,6 +840,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', + 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', @@ -929,6 +930,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/builtins.h', 'src/core/lib/surface/call.h', 'src/core/lib/surface/call_test_only.h', + 'src/core/lib/surface/call_trace.h', 'src/core/lib/surface/channel.h', 'src/core/lib/surface/channel_init.h', 'src/core/lib/surface/channel_stack_type.h', @@ -941,6 +943,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/server.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/transport/bdp_estimator.h', + 'src/core/lib/transport/call_fragments.h', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/handshaker.h', @@ -1703,6 +1706,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', + 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', @@ -1792,6 +1796,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/builtins.h', 'src/core/lib/surface/call.h', 'src/core/lib/surface/call_test_only.h', + 'src/core/lib/surface/call_trace.h', 'src/core/lib/surface/channel.h', 'src/core/lib/surface/channel_init.h', 'src/core/lib/surface/channel_stack_type.h', @@ -1804,6 +1809,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/server.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/transport/bdp_estimator.h', + 'src/core/lib/transport/call_fragments.h', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/handshaker.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index b7b83377ccf..e8ea7792d09 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -175,6 +175,7 @@ Pod::Spec.new do |s| ss.dependency 'BoringSSL-GRPC', '0.0.24' ss.dependency 'abseil/base/base', abseil_version ss.dependency 'abseil/base/core_headers', abseil_version + ss.dependency 'abseil/cleanup/cleanup', abseil_version ss.dependency 'abseil/container/flat_hash_map', abseil_version ss.dependency 'abseil/container/flat_hash_set', abseil_version ss.dependency 'abseil/container/inlined_vector', abseil_version @@ -1359,6 +1360,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', + 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', @@ -1535,6 +1537,8 @@ Pod::Spec.new do |s| 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', 'src/core/lib/surface/call_test_only.h', + 'src/core/lib/surface/call_trace.cc', + 'src/core/lib/surface/call_trace.h', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel.h', 'src/core/lib/surface/channel_init.cc', @@ -1562,6 +1566,8 @@ Pod::Spec.new do |s| 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', 'src/core/lib/transport/bdp_estimator.h', + 'src/core/lib/transport/call_fragments.cc', + 'src/core/lib/transport/call_fragments.h', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.cc', @@ -2330,6 +2336,7 @@ Pod::Spec.new do |s| 'src/core/lib/promise/latch.h', 'src/core/lib/promise/loop.h', 'src/core/lib/promise/map.h', + 'src/core/lib/promise/pipe.h', 'src/core/lib/promise/poll.h', 'src/core/lib/promise/promise.h', 'src/core/lib/promise/race.h', @@ -2419,6 +2426,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/builtins.h', 'src/core/lib/surface/call.h', 'src/core/lib/surface/call_test_only.h', + 'src/core/lib/surface/call_trace.h', 'src/core/lib/surface/channel.h', 'src/core/lib/surface/channel_init.h', 'src/core/lib/surface/channel_stack_type.h', @@ -2431,6 +2439,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/server.h', 'src/core/lib/surface/validate_metadata.h', 'src/core/lib/transport/bdp_estimator.h', + 'src/core/lib/transport/call_fragments.h', 'src/core/lib/transport/connectivity_state.h', 'src/core/lib/transport/error_utils.h', 'src/core/lib/transport/handshaker.h', diff --git a/grpc.gemspec b/grpc.gemspec index ecd3cc1168b..916945b0413 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -1271,6 +1271,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/promise/latch.h ) s.files += %w( src/core/lib/promise/loop.h ) s.files += %w( src/core/lib/promise/map.h ) + s.files += %w( src/core/lib/promise/pipe.h ) s.files += %w( src/core/lib/promise/poll.h ) s.files += %w( src/core/lib/promise/promise.h ) s.files += %w( src/core/lib/promise/race.h ) @@ -1447,6 +1448,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/surface/call_details.cc ) s.files += %w( src/core/lib/surface/call_log_batch.cc ) s.files += %w( src/core/lib/surface/call_test_only.h ) + s.files += %w( src/core/lib/surface/call_trace.cc ) + s.files += %w( src/core/lib/surface/call_trace.h ) s.files += %w( src/core/lib/surface/channel.cc ) s.files += %w( src/core/lib/surface/channel.h ) s.files += %w( src/core/lib/surface/channel_init.cc ) @@ -1474,6 +1477,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/surface/version.cc ) s.files += %w( src/core/lib/transport/bdp_estimator.cc ) s.files += %w( src/core/lib/transport/bdp_estimator.h ) + s.files += %w( src/core/lib/transport/call_fragments.cc ) + s.files += %w( src/core/lib/transport/call_fragments.h ) s.files += %w( src/core/lib/transport/connectivity_state.cc ) s.files += %w( src/core/lib/transport/connectivity_state.h ) s.files += %w( src/core/lib/transport/error_utils.cc ) @@ -1620,6 +1625,8 @@ Gem::Specification.new do |s| s.files += %w( third_party/abseil-cpp/absl/base/policy_checks.h ) s.files += %w( third_party/abseil-cpp/absl/base/port.h ) s.files += %w( third_party/abseil-cpp/absl/base/thread_annotations.h ) + s.files += %w( third_party/abseil-cpp/absl/cleanup/cleanup.h ) + s.files += %w( third_party/abseil-cpp/absl/cleanup/internal/cleanup.h ) s.files += %w( third_party/abseil-cpp/absl/container/fixed_array.h ) s.files += %w( third_party/abseil-cpp/absl/container/flat_hash_map.h ) s.files += %w( third_party/abseil-cpp/absl/container/flat_hash_set.h ) diff --git a/grpc.gyp b/grpc.gyp index 37b9f60deb5..87cbd782346 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -355,6 +355,7 @@ 'target_name': 'grpc', 'type': 'static_library', 'dependencies': [ + 'absl/cleanup:cleanup', 'absl/container:flat_hash_map', 'absl/container:flat_hash_set', 'absl/container:inlined_vector', @@ -1000,6 +1001,7 @@ 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', + 'src/core/lib/surface/call_trace.cc', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel_init.cc', 'src/core/lib/surface/channel_ping.cc', @@ -1015,6 +1017,7 @@ 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', + 'src/core/lib/transport/call_fragments.cc', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/handshaker.cc', @@ -1108,6 +1111,7 @@ 'target_name': 'grpc_unsecure', 'type': 'static_library', 'dependencies': [ + 'absl/cleanup:cleanup', 'absl/container:flat_hash_map', 'absl/container:flat_hash_set', 'absl/container:inlined_vector', @@ -1410,6 +1414,7 @@ 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', + 'src/core/lib/surface/call_trace.cc', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel_init.cc', 'src/core/lib/surface/channel_ping.cc', @@ -1425,6 +1430,7 @@ 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', + 'src/core/lib/transport/call_fragments.cc', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/handshaker.cc', @@ -1480,7 +1486,6 @@ 'target_name': 'grpc++', 'type': 'static_library', 'dependencies': [ - 'absl/cleanup:cleanup', 'grpc', ], 'sources': [ diff --git a/package.xml b/package.xml index 6e3ed42e7f0..f3566cb0a59 100644 --- a/package.xml +++ b/package.xml @@ -1253,6 +1253,7 @@ + @@ -1429,6 +1430,8 @@ + + @@ -1456,6 +1459,8 @@ + + @@ -1624,6 +1629,8 @@ + + diff --git a/src/core/ext/filters/channel_idle/channel_idle_filter.h b/src/core/ext/filters/channel_idle/channel_idle_filter.h index bc49a2a0174..696c92b2f20 100644 --- a/src/core/ext/filters/channel_idle/channel_idle_filter.h +++ b/src/core/ext/filters/channel_idle/channel_idle_filter.h @@ -35,6 +35,7 @@ #include "src/core/lib/gprpp/time.h" #include "src/core/lib/promise/activity.h" #include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/transport.h" diff --git a/src/core/ext/filters/fault_injection/fault_injection_filter.h b/src/core/ext/filters/fault_injection/fault_injection_filter.h index 611c3d7f45b..a96d35715c2 100644 --- a/src/core/ext/filters/fault_injection/fault_injection_filter.h +++ b/src/core/ext/filters/fault_injection/fault_injection_filter.h @@ -32,6 +32,7 @@ #include "src/core/lib/channel/promise_based_filter.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/transport.h" // Channel arg key for enabling parsing fault injection via method config. diff --git a/src/core/ext/filters/http/client/http_client_filter.cc b/src/core/ext/filters/http/client/http_client_filter.cc index 563adc63393..bb3ea724222 100644 --- a/src/core/ext/filters/http/client/http_client_filter.cc +++ b/src/core/ext/filters/http/client/http_client_filter.cc @@ -45,6 +45,7 @@ #include "src/core/lib/promise/seq.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/percent_encoding.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/status_conversion.h" #include "src/core/lib/transport/transport_fwd.h" #include "src/core/lib/transport/transport_impl.h" diff --git a/src/core/ext/filters/http/client/http_client_filter.h b/src/core/ext/filters/http/client/http_client_filter.h index 5a62d01053a..7d33c9dc64d 100644 --- a/src/core/ext/filters/http/client/http_client_filter.h +++ b/src/core/ext/filters/http/client/http_client_filter.h @@ -27,6 +27,7 @@ #include "src/core/lib/channel/promise_based_filter.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/slice/slice.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" diff --git a/src/core/ext/filters/http/client_authority_filter.cc b/src/core/ext/filters/http/client_authority_filter.cc index 9236b62debb..67d1de3458e 100644 --- a/src/core/ext/filters/http/client_authority_filter.cc +++ b/src/core/ext/filters/http/client_authority_filter.cc @@ -35,6 +35,7 @@ #include "src/core/lib/config/core_configuration.h" #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" namespace grpc_core { diff --git a/src/core/ext/filters/http/client_authority_filter.h b/src/core/ext/filters/http/client_authority_filter.h index 4f52e52f88a..1eed0ffa220 100644 --- a/src/core/ext/filters/http/client_authority_filter.h +++ b/src/core/ext/filters/http/client_authority_filter.h @@ -30,6 +30,7 @@ #include "src/core/lib/channel/promise_based_filter.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/slice/slice.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { diff --git a/src/core/ext/filters/http/server/http_server_filter.cc b/src/core/ext/filters/http/server/http_server_filter.cc index 2d0b44d611a..eb7ef5d3e97 100644 --- a/src/core/ext/filters/http/server/http_server_filter.cc +++ b/src/core/ext/filters/http/server/http_server_filter.cc @@ -40,6 +40,7 @@ #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/percent_encoding.h" #include "src/core/lib/slice/slice.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" namespace grpc_core { diff --git a/src/core/ext/filters/http/server/http_server_filter.h b/src/core/ext/filters/http/server/http_server_filter.h index b4fc7ecc496..f9d1a3081fb 100644 --- a/src/core/ext/filters/http/server/http_server_filter.h +++ b/src/core/ext/filters/http/server/http_server_filter.h @@ -27,6 +27,7 @@ #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/promise_based_filter.h" #include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc index dfa689f4afc..50e2af1e144 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc +++ b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc @@ -63,6 +63,7 @@ #include "src/core/lib/slice/slice.h" #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/uri/uri_parser.h" #include "src/cpp/server/load_reporter/constants.h" diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_filter.h b/src/core/ext/filters/load_reporting/server_load_reporting_filter.h index f786bfe0b9b..e63184c54b0 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_filter.h +++ b/src/core/ext/filters/load_reporting/server_load_reporting_filter.h @@ -30,6 +30,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/promise_based_filter.h" #include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { diff --git a/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc b/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc index f919147721a..c0ed6be7957 100644 --- a/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc +++ b/src/core/ext/filters/server_config_selector/server_config_selector_filter.cc @@ -39,6 +39,7 @@ #include "src/core/lib/promise/promise.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/service_config/service_config_call_data.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 4e9002b070a..4127e2a0faf 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1175,6 +1175,7 @@ static bool contains_non_ok_status(grpc_metadata_batch* batch) { static void log_metadata(const grpc_metadata_batch* md_batch, uint32_t id, bool is_client, bool is_initial) { + gpr_log(GPR_INFO, "--metadata--"); const std::string prefix = absl::StrCat( "HTTP:", id, is_initial ? ":HDR" : ":TRL", is_client ? ":CLI:" : ":SVR:"); md_batch->Log([&prefix](absl::string_view key, absl::string_view value) { diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.cc b/src/core/ext/transport/chttp2/transport/hpack_parser.cc index 3c0e5b0288c..fd4e821e7dd 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.cc @@ -1258,6 +1258,9 @@ void HPackParser::BeginFrame(grpc_metadata_batch* metadata_buffer, uint32_t metadata_size_limit, Boundary boundary, Priority priority, LogInfo log_info) { metadata_buffer_ = metadata_buffer; + if (metadata_buffer != nullptr) { + metadata_buffer->Set(GrpcStatusFromWire(), true); + } boundary_ = boundary; priority_ = priority; dynamic_table_updates_allowed_ = 2; diff --git a/src/core/lib/channel/channel_stack.cc b/src/core/lib/channel/channel_stack.cc index 4fb443c0c60..b73d38f6322 100644 --- a/src/core/lib/channel/channel_stack.cc +++ b/src/core/lib/channel/channel_stack.cc @@ -299,12 +299,12 @@ grpc_core::NextPromiseFactory ServerNext(grpc_channel_element* elem) { } // namespace grpc_core::ArenaPromise -grpc_channel_stack::MakeCallPromise(grpc_core::CallArgs call_args) { - if (is_client) { - return ClientNext(grpc_channel_stack_element(this, 0))( - std::move(call_args)); - } else { - return ServerNext(grpc_channel_stack_element(this, this->count - 1))( - std::move(call_args)); - } +grpc_channel_stack::MakeClientCallPromise(grpc_core::CallArgs call_args) { + return ClientNext(grpc_channel_stack_element(this, 0))(std::move(call_args)); +} + +grpc_core::ArenaPromise +grpc_channel_stack::MakeServerCallPromise(grpc_core::CallArgs call_args) { + return ServerNext(grpc_channel_stack_element(this, this->count - 1))( + std::move(call_args)); } diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index 717035efbe1..1167772502e 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -70,6 +70,7 @@ #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/transport.h" struct grpc_channel_element_args { @@ -202,7 +203,6 @@ struct grpc_call_element { guarantees they live within a single malloc() allocation */ struct grpc_channel_stack { grpc_stream_refcount refcount; - bool is_client; size_t count; /* Memory required for a call stack (computed at channel stack initialization) */ @@ -225,8 +225,10 @@ struct grpc_channel_stack { return grpc_core::RefCountedPtr(this); } - grpc_core::ArenaPromise MakeCallPromise( - grpc_core::CallArgs call_args); + grpc_core::ArenaPromise + MakeClientCallPromise(grpc_core::CallArgs call_args); + grpc_core::ArenaPromise + MakeServerCallPromise(grpc_core::CallArgs call_args); }; /* A call stack tracks a set of related filters for one call, and guarantees diff --git a/src/core/lib/channel/channel_stack_builder.h b/src/core/lib/channel/channel_stack_builder.h index 71343f33fb6..f08efa72230 100644 --- a/src/core/lib/channel/channel_stack_builder.h +++ b/src/core/lib/channel/channel_stack_builder.h @@ -71,6 +71,11 @@ class ChannelStackBuilder { // Mutable vector of proposed stack entries. std::vector* mutable_stack() { return &stack_; } + // Immutable vector of proposed stack entries. + const std::vector& stack() const { + return stack_; + } + // The type of channel stack being built. grpc_channel_stack_type channel_stack_type() const { return type_; } @@ -80,6 +85,11 @@ class ChannelStackBuilder { // Helper to add a filter to the end of the stack. void AppendFilter(const grpc_channel_filter* filter); + // Determine whether a promise-based call stack is able to be built. + // Iterates each filter and ensures that there's a promise factory there. + // This will go away once the promise conversion is completed. + virtual bool IsPromising() const = 0; + // Build the channel stack. // After success, *result holds the new channel stack, // prefix_bytes are allocated before the channel stack, diff --git a/src/core/lib/channel/channel_stack_builder_impl.cc b/src/core/lib/channel/channel_stack_builder_impl.cc index dc1ab2abca3..db5d34ba6ea 100644 --- a/src/core/lib/channel/channel_stack_builder_impl.cc +++ b/src/core/lib/channel/channel_stack_builder_impl.cc @@ -22,6 +22,7 @@ #include +#include #include #include "absl/status/status.h" @@ -30,21 +31,39 @@ #include #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/iomgr/error.h" +#include "src/core/lib/surface/call_trace.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { +bool ChannelStackBuilderImpl::IsPromising() const { + for (const auto* filter : stack()) { + if (filter->make_call_promise == nullptr) return false; + } + return true; +} + absl::StatusOr> ChannelStackBuilderImpl::Build() { - auto* stack = mutable_stack(); + std::vector stack; + const bool is_promising = IsPromising(); + + for (const auto* filter : this->stack()) { + if (is_promising && grpc_call_trace.enabled()) { + stack.push_back(PromiseTracingFilterFor(filter)); + } + stack.push_back(filter); + } // calculate the size of the channel stack size_t channel_stack_size = - grpc_channel_stack_size(stack->data(), stack->size()); + grpc_channel_stack_size(stack.data(), stack.size()); // allocate memory auto* channel_stack = @@ -72,7 +91,7 @@ ChannelStackBuilderImpl::Build() { grpc_channel_stack_destroy(stk); gpr_free(stk); }, - channel_stack, stack->data(), stack->size(), final_args, name(), + channel_stack, stack.data(), stack.size(), final_args, name(), channel_stack); if (!error.ok()) { @@ -83,7 +102,7 @@ ChannelStackBuilderImpl::Build() { } // run post-initialization functions - for (size_t i = 0; i < stack->size(); i++) { + for (size_t i = 0; i < stack.size(); i++) { auto* elem = grpc_channel_stack_element(channel_stack, i); elem->filter->post_init_channel_elem(channel_stack, elem); } diff --git a/src/core/lib/channel/channel_stack_builder_impl.h b/src/core/lib/channel/channel_stack_builder_impl.h index b466c5e42e2..e206f0400e4 100644 --- a/src/core/lib/channel/channel_stack_builder_impl.h +++ b/src/core/lib/channel/channel_stack_builder_impl.h @@ -34,6 +34,8 @@ class ChannelStackBuilderImpl final : public ChannelStackBuilder { public: using ChannelStackBuilder::ChannelStackBuilder; + bool IsPromising() const override; + // Build the channel stack. // After success, *result holds the new channel stack, // prefix_bytes are allocated before the channel stack, diff --git a/src/core/lib/channel/connected_channel.cc b/src/core/lib/channel/connected_channel.cc index e9629b928c9..ed77e8cdcc4 100644 --- a/src/core/lib/channel/connected_channel.cc +++ b/src/core/lib/channel/connected_channel.cc @@ -20,21 +20,57 @@ #include "src/core/lib/channel/connected_channel.h" +#include +#include + +#include +#include +#include +#include +#include + +#include "absl/base/thread_annotations.h" #include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" #include #include #include #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/channel/context.h" +#include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/alloc.h" +#include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/match.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/polling_entity.h" +#include "src/core/lib/promise/activity.h" +#include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/poll.h" +#include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/surface/call.h" +#include "src/core/lib/surface/call_trace.h" +#include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/transport/call_fragments.h" +#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport_fwd.h" +#include "src/core/lib/transport/transport_impl.h" #define MAX_BUFFER_LENGTH 8192 @@ -205,39 +241,661 @@ static void connected_channel_get_channel_info( grpc_channel_element* /*elem*/, const grpc_channel_info* /*channel_info*/) { } -const grpc_channel_filter grpc_connected_filter = { - connected_channel_start_transport_stream_op_batch, - nullptr, - connected_channel_start_transport_op, - sizeof(call_data), - connected_channel_init_call_elem, - set_pollset_or_pollset_set, - connected_channel_destroy_call_elem, - sizeof(channel_data), - connected_channel_init_channel_elem, - [](grpc_channel_stack* channel_stack, grpc_channel_element* elem) { - /* HACK(ctiller): increase call stack size for the channel to make space - for channel data. We need a cleaner (but performant) way to do this, - and I'm not sure what that is yet. - This is only "safe" because call stacks place no additional data after - the last call element, and the last call element MUST be the connected - channel. */ - channel_stack->call_stack_size += grpc_transport_stream_size( - static_cast(elem->channel_data)->transport); - }, - connected_channel_destroy_channel_elem, - connected_channel_get_channel_info, - "connected", +namespace grpc_core { +namespace { + +class ClientStream : public Orphanable { + public: + ClientStream(grpc_transport* transport, CallArgs call_args) + : transport_(transport), + stream_(nullptr, StreamDeleter(this)), + server_initial_metadata_latch_(call_args.server_initial_metadata), + client_to_server_messages_(call_args.outgoing_messages), + server_to_client_messages_(call_args.incoming_messages), + client_initial_metadata_(std::move(call_args.client_initial_metadata)) { + call_context_->IncrementRefCount("client_stream"); + GRPC_STREAM_REF_INIT( + &stream_refcount_, 1, + [](void* p, grpc_error_handle) { + static_cast(p)->BeginDestroy(); + }, + this, "client_stream"); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sInitImpl: intitial_metadata=%s", + Activity::current()->DebugTag().c_str(), + client_initial_metadata_->DebugString().c_str()); + } + } + + void Orphan() override { + bool finished; + { + MutexLock lock(&mu_); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sDropStream: %s", + Activity::current()->DebugTag().c_str(), + ActiveOpsString().c_str()); + } + finished = finished_; + } + // If we hadn't already observed the stream to be finished, we need to + // cancel it at the transport. + if (!finished) { + IncrementRefCount("shutdown client stream"); + auto* cancel_op = + GetContext()->New(); + cancel_op->cancel_stream = true; + cancel_op->payload = &batch_payload_; + auto* stream = stream_.get(); + cancel_op->on_complete = NewClosure( + [this](grpc_error_handle) { Unref("shutdown client stream"); }); + batch_payload_.cancel_stream.cancel_error = GRPC_ERROR_CANCELLED; + grpc_transport_perform_stream_op(transport_, stream, cancel_op); + } + Unref("orphan client stream"); + } + + void IncrementRefCount(const char* reason) { +#ifndef NDEBUG + grpc_stream_ref(&stream_refcount_, reason); +#else + (void)reason; + grpc_stream_ref(&stream_refcount_); +#endif + } + + void Unref(const char* reason) { +#ifndef NDEBUG + grpc_stream_unref(&stream_refcount_, reason); +#else + (void)reason; + grpc_stream_unref(&stream_refcount_); +#endif + } + + void BeginDestroy() { + if (stream_ != nullptr) { + stream_.reset(); + } else { + StreamDestroyed(); + } + } + + Poll PollOnce() { + MutexLock lock(&mu_); + GPR_ASSERT(!finished_); + + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sPollConnectedChannel: %s", + Activity::current()->DebugTag().c_str(), + ActiveOpsString().c_str()); + } + + auto push_recv_message = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + recv_message_state_ = PendingReceiveMessage{}; + auto& pending_recv_message = + absl::get(recv_message_state_); + memset(&recv_message_, 0, sizeof(recv_message_)); + recv_message_.payload = &batch_payload_; + recv_message_.on_complete = nullptr; + recv_message_.recv_message = true; + batch_payload_.recv_message.recv_message = &pending_recv_message.payload; + batch_payload_.recv_message.flags = &pending_recv_message.flags; + batch_payload_.recv_message.call_failed_before_recv_message = nullptr; + batch_payload_.recv_message.recv_message_ready = + &recv_message_batch_done_; + IncrementRefCount("recv_message"); + recv_message_waker_ = Activity::current()->MakeOwningWaker(); + push_recv_message_ = true; + SchedulePush(); + }; + + if (!std::exchange(requested_metadata_, true)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sPollConnectedChannel: requesting metadata", + Activity::current()->DebugTag().c_str()); + } + stream_.reset(static_cast( + GetContext()->Alloc(transport_->vtable->sizeof_stream))); + grpc_transport_init_stream(transport_, stream_.get(), &stream_refcount_, + nullptr, GetContext()); + grpc_transport_set_pops(transport_, stream_.get(), + GetContext()->polling_entity()); + memset(&metadata_, 0, sizeof(metadata_)); + metadata_.send_initial_metadata = true; + metadata_.recv_initial_metadata = true; + metadata_.recv_trailing_metadata = true; + metadata_.payload = &batch_payload_; + metadata_.on_complete = &metadata_batch_done_; + batch_payload_.send_initial_metadata.send_initial_metadata = + client_initial_metadata_.get(); + batch_payload_.send_initial_metadata.peer_string = + GetContext()->peer_string_atm_ptr(); + server_initial_metadata_ = + GetContext()->MakeServerMetadata(); + batch_payload_.recv_initial_metadata.recv_initial_metadata = + server_initial_metadata_.get(); + batch_payload_.recv_initial_metadata.recv_initial_metadata_ready = + &recv_initial_metadata_ready_; + batch_payload_.recv_initial_metadata.trailing_metadata_available = + nullptr; + batch_payload_.recv_initial_metadata.peer_string = nullptr; + server_trailing_metadata_ = + GetContext()->MakeClientMetadata(); + batch_payload_.recv_trailing_metadata.recv_trailing_metadata = + server_trailing_metadata_.get(); + batch_payload_.recv_trailing_metadata.collect_stats = + &GetContext()->call_stats()->transport_stream_stats; + batch_payload_.recv_trailing_metadata.recv_trailing_metadata_ready = + &recv_trailing_metadata_ready_; + push_metadata_ = true; + IncrementRefCount("metadata_batch_done"); + IncrementRefCount("initial_metadata_ready"); + IncrementRefCount("trailing_metadata_ready"); + initial_metadata_waker_ = Activity::current()->MakeOwningWaker(); + trailing_metadata_waker_ = Activity::current()->MakeOwningWaker(); + SchedulePush(); + } + if (absl::holds_alternative(send_message_state_)) { + message_to_send_.reset(); + } + if (absl::holds_alternative(send_message_state_)) { + message_to_send_.reset(); + send_message_state_ = client_to_server_messages_->Next(); + } + if (auto* next = absl::get_if::NextType>( + &send_message_state_)) { + auto r = (*next)(); + if (auto* p = absl::get_if>(&r)) { + memset(&send_message_, 0, sizeof(send_message_)); + send_message_.payload = &batch_payload_; + send_message_.on_complete = &send_message_batch_done_; + // No value => half close from above. + if (p->has_value()) { + message_to_send_ = std::move(**p); + send_message_state_ = SendMessageToTransport{}; + send_message_.send_message = true; + batch_payload_.send_message.send_message = + message_to_send_->payload(); + batch_payload_.send_message.flags = message_to_send_->flags(); + } else { + GPR_ASSERT(!absl::holds_alternative(send_message_state_)); + client_trailing_metadata_ = + GetContext()->MakeClientMetadata(); + send_message_state_ = Closed{}; + send_message_.send_trailing_metadata = true; + batch_payload_.send_trailing_metadata.send_trailing_metadata = + client_trailing_metadata_.get(); + batch_payload_.send_trailing_metadata.sent = nullptr; + } + IncrementRefCount("send_message"); + send_message_waker_ = Activity::current()->MakeOwningWaker(); + push_send_message_ = true; + SchedulePush(); + } + } + if (auto* pending = + absl::get_if(&recv_message_state_)) { + if (pending->received) { + if (pending->payload.has_value()) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sRecvMessageBatchDone: received payload of %" PRIdPTR + " bytes", + recv_message_waker_.ActivityDebugTag().c_str(), + pending->payload->Length()); + } + recv_message_state_ = server_to_client_messages_->Push( + GetContext()->MakeMessage( + std::move(*pending->payload), pending->flags)); + } else { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sRecvMessageBatchDone: received no payload", + recv_message_waker_.ActivityDebugTag().c_str()); + } + recv_message_state_ = Closed{}; + std::exchange(server_to_client_messages_, nullptr)->Close(); + } + } + } + if (server_initial_metadata_state_ == + ServerInitialMetadataState::kReceivedButNotSet) { + server_initial_metadata_state_ = ServerInitialMetadataState::kSet; + server_initial_metadata_latch_->Set(server_initial_metadata_.get()); + } + if (absl::holds_alternative(recv_message_state_)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sPollConnectedChannel: requesting message", + Activity::current()->DebugTag().c_str()); + } + push_recv_message(); + } + if (server_initial_metadata_state_ == ServerInitialMetadataState::kSet && + !absl::holds_alternative::PushType>( + recv_message_state_) && + std::exchange(queued_trailing_metadata_, false)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sPollConnectedChannel: finished request, returning: {%s}; " + "active_ops: %s", + Activity::current()->DebugTag().c_str(), + server_trailing_metadata_->DebugString().c_str(), + ActiveOpsString().c_str()); + } + finished_ = true; + return ServerMetadataHandle(std::move(server_trailing_metadata_)); + } + if (auto* push = absl::get_if::PushType>( + &recv_message_state_)) { + auto r = (*push)(); + if (bool* result = absl::get_if(&r)) { + if (*result) { + if (!finished_) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sPollConnectedChannel: pushed message; requesting next", + Activity::current()->DebugTag().c_str()); + } + push_recv_message(); + } else { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sPollConnectedChannel: pushed message and finished; " + "marking closed", + Activity::current()->DebugTag().c_str()); + } + recv_message_state_ = Closed{}; + } + } else { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sPollConnectedChannel: failed to push message; marking " + "closed", + Activity::current()->DebugTag().c_str()); + } + recv_message_state_ = Closed{}; + } + } + } + return Pending{}; + } + + void RecvInitialMetadataReady(grpc_error_handle error) { + GPR_ASSERT(error == GRPC_ERROR_NONE); + { + MutexLock lock(&mu_); + server_initial_metadata_state_ = + ServerInitialMetadataState::kReceivedButNotSet; + initial_metadata_waker_.Wakeup(); + } + Unref("initial_metadata_ready"); + } + + void RecvTrailingMetadataReady(grpc_error_handle error) { + GPR_ASSERT(error == GRPC_ERROR_NONE); + { + MutexLock lock(&mu_); + queued_trailing_metadata_ = true; + trailing_metadata_waker_.Wakeup(); + } + Unref("trailing_metadata_ready"); + } + + void MetadataBatchDone(grpc_error_handle error) { + GPR_ASSERT(error == GRPC_ERROR_NONE); + Unref("metadata_batch_done"); + } + + void SendMessageBatchDone(grpc_error_handle error) { + { + MutexLock lock(&mu_); + if (error != GRPC_ERROR_NONE) { + // Note that we're in error here, the call will be closed by the + // transport in a moment, and we'll return from the promise with an + // error - so we don't need to do any extra work to close out pipes or + // the like. + send_message_state_ = Closed{}; + } + if (!absl::holds_alternative(send_message_state_)) { + send_message_state_ = Idle{}; + } + send_message_waker_.Wakeup(); + } + Unref("send_message"); + } + + void RecvMessageBatchDone(grpc_error_handle error) { + { + MutexLock lock(&mu_); + if (error != GRPC_ERROR_NONE) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sRecvMessageBatchDone: error=%s", + recv_message_waker_.ActivityDebugTag().c_str(), + grpc_error_std_string(error).c_str()); + } + } else if (absl::holds_alternative(recv_message_state_)) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sRecvMessageBatchDone: already closed, ignoring", + recv_message_waker_.ActivityDebugTag().c_str()); + } + } else { + auto pending = + absl::get_if(&recv_message_state_); + GPR_ASSERT(pending != nullptr); + GPR_ASSERT(pending->received == false); + pending->received = true; + } + recv_message_waker_.Wakeup(); + } + Unref("recv_message"); + } + + // Called from outside the activity to push work down to the transport. + void Push() { + auto do_push = [this](grpc_transport_stream_op_batch* batch) { + if (stream_ != nullptr) { + grpc_transport_perform_stream_op(transport_, stream_.get(), batch); + } else { + grpc_transport_stream_op_batch_finish_with_failure_from_transport( + batch, GRPC_ERROR_CANCELLED); + } + }; + bool push_metadata; + bool push_send_message; + bool push_recv_message; + { + MutexLock lock(&mu_); + push_metadata = std::exchange(push_metadata_, false); + push_send_message = std::exchange(push_send_message_, false); + push_recv_message = std::exchange(push_recv_message_, false); + scheduled_push_ = false; + } + if (push_metadata) do_push(&metadata_); + if (push_send_message) do_push(&send_message_); + if (push_recv_message) do_push(&recv_message_); + Unref("push"); + } + + void StreamDestroyed() { + call_context_->RunInContext([this] { + auto* cc = call_context_; + this->~ClientStream(); + cc->Unref("child_stream"); + }); + } + + private: + struct Idle {}; + struct Closed {}; + struct SendMessageToTransport {}; + + enum class ServerInitialMetadataState : uint8_t { + // Initial metadata has not been received from the server. + kNotReceived, + // Initial metadata has been received from the server via the transport, but + // has not yet been set on the latch to publish it up the call stack. + kReceivedButNotSet, + // Initial metadata has been received from the server via the transport and + // has been set on the latch to publish it up the call stack. + kSet, + }; + + class StreamDeleter { + public: + explicit StreamDeleter(ClientStream* impl) : impl_(impl) {} + void operator()(grpc_stream* stream) const { + if (stream == nullptr) return; + grpc_transport_destroy_stream(impl_->transport_, stream, + &impl_->stream_destroyed_); + } + + private: + ClientStream* impl_; + }; + using StreamPtr = std::unique_ptr; + + void SchedulePush() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (std::exchange(scheduled_push_, true)) return; + IncrementRefCount("push"); + ExecCtx::Run(DEBUG_LOCATION, &push_, GRPC_ERROR_NONE); + } + + std::string ActiveOpsString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + std::vector ops; + if (finished_) ops.push_back("FINISHED"); + // Pushes + std::vector pushes; + if (push_metadata_) pushes.push_back("metadata"); + if (push_send_message_) pushes.push_back("send_message"); + if (push_recv_message_) pushes.push_back("recv_message"); + if (!pushes.empty()) { + ops.push_back( + absl::StrCat(scheduled_push_ ? "push:" : "unscheduled-push:", + absl::StrJoin(pushes, ","))); + } else if (scheduled_push_) { + ops.push_back("push:nothing"); + } + // Results from transport + std::vector queued; + if (server_initial_metadata_state_ == + ServerInitialMetadataState::kReceivedButNotSet) { + queued.push_back("initial_metadata"); + } + if (queued_trailing_metadata_) queued.push_back("trailing_metadata"); + if (!queued.empty()) { + ops.push_back(absl::StrCat("queued:", absl::StrJoin(queued, ","))); + } + // Send message + std::string send_message_state = SendMessageString(); + if (send_message_state != "WAITING") { + ops.push_back(absl::StrCat("send_message:", send_message_state)); + } + // Receive message + std::string recv_message_state = RecvMessageString(); + if (recv_message_state != "IDLE") { + ops.push_back(absl::StrCat("recv_message:", recv_message_state)); + } + return absl::StrJoin(ops, " "); + } + + std::string SendMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return Match( + send_message_state_, [](Idle) -> std::string { return "IDLE"; }, + [](Closed) -> std::string { return "CLOSED"; }, + [](const PipeReceiver::NextType&) -> std::string { + return "WAITING"; + }, + [](SendMessageToTransport) -> std::string { return "SENDING"; }); + } + + std::string RecvMessageString() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return Match( + recv_message_state_, [](Idle) -> std::string { return "IDLE"; }, + [](Closed) -> std::string { return "CLOSED"; }, + [](const PendingReceiveMessage&) -> std::string { return "WAITING"; }, + [](const absl::optional& message) -> std::string { + return absl::StrCat( + "READY:", message.has_value() + ? absl::StrCat((*message)->payload()->Length(), "b") + : "EOS"); + }, + [](const PipeSender::PushType&) -> std::string { + return "PUSHING"; + }); + } + + Mutex mu_; + bool requested_metadata_ = false; + bool push_metadata_ ABSL_GUARDED_BY(mu_) = false; + bool push_send_message_ ABSL_GUARDED_BY(mu_) = false; + bool push_recv_message_ ABSL_GUARDED_BY(mu_) = false; + bool scheduled_push_ ABSL_GUARDED_BY(mu_) = false; + ServerInitialMetadataState server_initial_metadata_state_ + ABSL_GUARDED_BY(mu_) = ServerInitialMetadataState::kNotReceived; + bool queued_trailing_metadata_ ABSL_GUARDED_BY(mu_) = false; + bool finished_ ABSL_GUARDED_BY(mu_) = false; + CallContext* const call_context_{GetContext()}; + Waker initial_metadata_waker_ ABSL_GUARDED_BY(mu_); + Waker trailing_metadata_waker_ ABSL_GUARDED_BY(mu_); + Waker send_message_waker_ ABSL_GUARDED_BY(mu_); + Waker recv_message_waker_ ABSL_GUARDED_BY(mu_); + grpc_transport* const transport_; + grpc_stream_refcount stream_refcount_; + StreamPtr stream_; + Latch* server_initial_metadata_latch_; + PipeReceiver* client_to_server_messages_; + PipeSender* server_to_client_messages_; + MessageHandle message_to_send_ ABSL_GUARDED_BY(mu_); + absl::variant::NextType, + SendMessageToTransport> + send_message_state_ ABSL_GUARDED_BY(mu_); + struct PendingReceiveMessage { + absl::optional payload; + uint32_t flags; + bool received = false; + }; + absl::variant::PushType> + recv_message_state_ ABSL_GUARDED_BY(mu_); + grpc_closure recv_initial_metadata_ready_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + grpc_closure recv_trailing_metadata_ready_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + grpc_closure push_ = MakeMemberClosure( + this, DEBUG_LOCATION); + ClientMetadataHandle client_initial_metadata_; + ClientMetadataHandle client_trailing_metadata_; + ServerMetadataHandle server_initial_metadata_; + ServerMetadataHandle server_trailing_metadata_; + grpc_transport_stream_op_batch metadata_; + grpc_closure metadata_batch_done_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + grpc_transport_stream_op_batch send_message_; + grpc_closure send_message_batch_done_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + grpc_closure recv_message_batch_done_ = + MakeMemberClosure( + this, DEBUG_LOCATION); + grpc_transport_stream_op_batch recv_message_; + grpc_transport_stream_op_batch_payload batch_payload_{ + GetContext()}; + grpc_closure stream_destroyed_ = + MakeMemberClosure( + this, DEBUG_LOCATION); }; +class ClientConnectedCallPromise { + public: + ClientConnectedCallPromise(grpc_transport* transport, CallArgs call_args) + : impl_(GetContext()->New(transport, + std::move(call_args))) {} + + ClientConnectedCallPromise(const ClientConnectedCallPromise&) = delete; + ClientConnectedCallPromise& operator=(const ClientConnectedCallPromise&) = + delete; + ClientConnectedCallPromise(ClientConnectedCallPromise&& other) noexcept + : impl_(std::exchange(other.impl_, nullptr)) {} + ClientConnectedCallPromise& operator=( + ClientConnectedCallPromise&& other) noexcept { + impl_ = std::move(other.impl_); + return *this; + } + + static ArenaPromise Make(grpc_transport* transport, + CallArgs call_args) { + return ClientConnectedCallPromise(transport, std::move(call_args)); + } + + Poll operator()() { return impl_->PollOnce(); } + + private: + OrphanablePtr impl_; +}; + +template (*make_call_promise)( + grpc_transport*, CallArgs)> +grpc_channel_filter MakeConnectedFilter() { + // Create a vtable that contains both the legacy call methods (for filter + // stack based calls) and the new promise based method for creating promise + // based calls (the latter iff make_call_promise != nullptr). + // In this way the filter can be inserted into either kind of channel stack, + // and only if all the filters in the stack are promise based will the call + // be promise based. + return { + connected_channel_start_transport_stream_op_batch, + make_call_promise == nullptr + ? nullptr + : +[](grpc_channel_element* elem, CallArgs call_args, + NextPromiseFactory) { + grpc_transport* transport = + static_cast(elem->channel_data)->transport; + return make_call_promise(transport, std::move(call_args)); + }, + connected_channel_start_transport_op, + sizeof(call_data), + connected_channel_init_call_elem, + set_pollset_or_pollset_set, + connected_channel_destroy_call_elem, + sizeof(channel_data), + connected_channel_init_channel_elem, + +[](grpc_channel_stack* channel_stack, grpc_channel_element* elem) { + /* HACK(ctiller): increase call stack size for the channel to make space + for channel data. We need a cleaner (but performant) way to do this, + and I'm not sure what that is yet. + This is only "safe" because call stacks place no additional data + after the last call element, and the last call element MUST be the + connected channel. */ + channel_stack->call_stack_size += grpc_transport_stream_size( + static_cast(elem->channel_data)->transport); + }, + connected_channel_destroy_channel_elem, + connected_channel_get_channel_info, + "connected", + }; +} + +ArenaPromise MakeTransportCallPromise( + grpc_transport* transport, CallArgs call_args) { + return transport->vtable->make_call_promise(transport, std::move(call_args)); +} + +const grpc_channel_filter kPromiseBasedTransportFilter = + MakeConnectedFilter(); + +const grpc_channel_filter kClientEmulatedFilter = + MakeConnectedFilter(); + +const grpc_channel_filter kNoPromiseFilter = MakeConnectedFilter(); + +} // namespace +} // namespace grpc_core + bool grpc_add_connected_filter(grpc_core::ChannelStackBuilder* builder) { grpc_transport* t = builder->transport(); GPR_ASSERT(t != nullptr); - builder->AppendFilter(&grpc_connected_filter); + // Choose the right vtable for the connected filter. + // We can't know promise based call or not here (that decision needs the + // collaboration of all of the filters on the channel, and we don't want + // ordering constraints on when we add filters). + // We can know if this results in a promise based call how we'll create our + // promise (if indeed we can), and so that is the choice made here. + if (t->vtable->make_call_promise != nullptr) { + // Option 1, and our ideal: the transport supports promise based calls, and + // so we simply use the transport directly. + builder->AppendFilter(&grpc_core::kPromiseBasedTransportFilter); + } else if (grpc_channel_stack_type_is_client(builder->channel_stack_type())) { + // Option 2: the transport does not support promise based calls, but we're + // on the client and so we have an implementation that we can use to convert + // to batches. + builder->AppendFilter(&grpc_core::kClientEmulatedFilter); + } else { + // Option 3: the transport does not support promise based calls, and we're + // on the server so we can't construct promise based calls just yet. + builder->AppendFilter(&grpc_core::kNoPromiseFilter); + } return true; } - -grpc_stream* grpc_connected_channel_get_stream(grpc_call_element* elem) { - call_data* calld = static_cast(elem->call_data); - return TRANSPORT_STREAM_FROM_CALL_DATA(calld); -} diff --git a/src/core/lib/channel/connected_channel.h b/src/core/lib/channel/connected_channel.h index ea1428310e9..a139907b72b 100644 --- a/src/core/lib/channel/connected_channel.h +++ b/src/core/lib/channel/connected_channel.h @@ -24,13 +24,9 @@ #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/channel_stack_builder.h" -#include "src/core/lib/transport/transport.h" extern const grpc_channel_filter grpc_connected_filter; bool grpc_add_connected_filter(grpc_core::ChannelStackBuilder* builder); -/* Debug helper to dig the transport stream out of a call element */ -grpc_stream* grpc_connected_channel_get_stream(grpc_call_element* elem); - #endif /* GRPC_CORE_LIB_CHANNEL_CONNECTED_CHANNEL_H */ diff --git a/src/core/lib/channel/promise_based_filter.cc b/src/core/lib/channel/promise_based_filter.cc index 5c99adc05e4..068187b0325 100644 --- a/src/core/lib/channel/promise_based_filter.cc +++ b/src/core/lib/channel/promise_based_filter.cc @@ -665,7 +665,7 @@ void ClientCallData::StartPromise(Flusher* flusher) { promise_ = filter->MakeCallPromise( CallArgs{WrapMetadata(send_initial_metadata_batch_->payload ->send_initial_metadata.send_initial_metadata), - server_initial_metadata_latch()}, + server_initial_metadata_latch(), nullptr, nullptr}, [this](CallArgs call_args) { return MakeNextPromise(std::move(call_args)); }); @@ -1154,12 +1154,12 @@ void ServerCallData::RecvInitialMetadataReady(grpc_error_handle error) { ScopedContext context(this); // Construct the promise. ChannelFilter* filter = static_cast(elem()->channel_data); - promise_ = - filter->MakeCallPromise(CallArgs{WrapMetadata(recv_initial_metadata_), - server_initial_metadata_latch()}, - [this](CallArgs call_args) { - return MakeNextPromise(std::move(call_args)); - }); + promise_ = filter->MakeCallPromise( + CallArgs{WrapMetadata(recv_initial_metadata_), + server_initial_metadata_latch(), nullptr, nullptr}, + [this](CallArgs call_args) { + return MakeNextPromise(std::move(call_args)); + }); // Poll once. WakeInsideCombiner(&flusher); if (auto* closure = diff --git a/src/core/lib/channel/promise_based_filter.h b/src/core/lib/channel/promise_based_filter.h index 28beb9179c8..7a87a091303 100644 --- a/src/core/lib/channel/promise_based_filter.h +++ b/src/core/lib/channel/promise_based_filter.h @@ -19,6 +19,9 @@ // promise-style. Most of this will be removed once the promises conversion is // completed. +// TODO(ctiller): When removing this file, also reduce the number of *'s on the +// server initial metadata latch. + #include #include @@ -27,6 +30,7 @@ #include #include #include +#include #include #include "absl/container/inlined_vector.h" @@ -56,6 +60,7 @@ #include "src/core/lib/promise/latch.h" #include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -146,6 +151,8 @@ class BaseCallData : public Activity, private Wakeable { Waker MakeNonOwningWaker() final; Waker MakeOwningWaker() final; + std::string ActivityDebugTag() const override { return DebugTag(); } + void Finalize(const grpc_call_final_info* final_info) { finalization_.Run(final_info); } @@ -234,13 +241,13 @@ class BaseCallData : public Activity, private Wakeable { grpc_transport_stream_op_batch* batch_; }; - static MetadataHandle WrapMetadata( + static FragmentHandle WrapMetadata( grpc_metadata_batch* p) { - return MetadataHandle(p); + return FragmentHandle(p, false); } static grpc_metadata_batch* UnwrapMetadata( - MetadataHandle p) { + FragmentHandle p) { return p.Unwrap(); } @@ -311,8 +318,8 @@ class ClientCallData : public BaseCallData { kQueued, // We've forwarded the op to the next filter. kForwarded, - // The op has completed from below, but we haven't yet forwarded it up (the - // promise gets to interject and mutate it). + // The op has completed from below, but we haven't yet forwarded it up + // (the promise gets to interject and mutate it). kComplete, // We've called the recv_metadata_ready callback from the original // recv_trailing_metadata op that was presented to us. diff --git a/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h b/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h index f78991d2d7b..11cdf903e74 100644 --- a/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h +++ b/src/core/lib/event_engine/posix_engine/tcp_socket_utils.h @@ -25,7 +25,6 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "absl/utility/utility.h" #include #include @@ -74,7 +73,7 @@ struct PosixTcpOptions { PosixTcpOptions() = default; // Move ctor PosixTcpOptions(PosixTcpOptions&& other) noexcept { - socket_mutator = absl::exchange(other.socket_mutator, nullptr); + socket_mutator = std::exchange(other.socket_mutator, nullptr); resource_quota = std::move(other.resource_quota); CopyIntegerOptions(other); } @@ -83,7 +82,7 @@ struct PosixTcpOptions { if (socket_mutator != nullptr) { grpc_socket_mutator_unref(socket_mutator); } - socket_mutator = absl::exchange(other.socket_mutator, nullptr); + socket_mutator = std::exchange(other.socket_mutator, nullptr); resource_quota = std::move(other.resource_quota); CopyIntegerOptions(other); return *this; diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index b065f0d66a1..667c25716f7 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -48,6 +48,9 @@ const char* const description_event_engine_client = "Use EventEngine clients instead of iomgr's grpc_tcp_client"; const char* const description_monitoring_experiment = "Placeholder experiment to prove/disprove our monitoring is working"; +const char* const description_promise_based_client_call = + "If set, use the new gRPC promise based call code when it's appropriate " + "(ie when all filters in a stack are promise based)"; #ifdef NDEBUG const bool kDefaultForDebugOnly = false; #else @@ -73,6 +76,7 @@ const ExperimentMetadata g_experiment_metadata[] = { kDefaultForDebugOnly}, {"event_engine_client", description_event_engine_client, false}, {"monitoring_experiment", description_monitoring_experiment, true}, + {"promise_based_client_call", description_promise_based_client_call, false}, }; } // namespace grpc_core diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index 18131f89b17..6a0d44b2c92 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -42,6 +42,9 @@ inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() { inline bool IsNewHpackHuffmanDecoderEnabled() { return IsExperimentEnabled(8); } inline bool IsEventEngineClientEnabled() { return IsExperimentEnabled(9); } inline bool IsMonitoringExperimentEnabled() { return IsExperimentEnabled(10); } +inline bool IsPromiseBasedClientCallEnabled() { + return IsExperimentEnabled(11); +} struct ExperimentMetadata { const char* name; @@ -49,7 +52,7 @@ struct ExperimentMetadata { bool default_value; }; -constexpr const size_t kNumExperiments = 11; +constexpr const size_t kNumExperiments = 12; extern const ExperimentMetadata g_experiment_metadata[kNumExperiments]; } // namespace grpc_core diff --git a/src/core/lib/experiments/experiments.yaml b/src/core/lib/experiments/experiments.yaml index 18ccd978c03..dd9ba5d5cb5 100644 --- a/src/core/lib/experiments/experiments.yaml +++ b/src/core/lib/experiments/experiments.yaml @@ -119,3 +119,11 @@ expiry: 2022/10/01 owner: ctiller@google.com test_tags: [] +- name: promise_based_client_call + description: + If set, use the new gRPC promise based call code when it's appropriate + (ie when all filters in a stack are promise based) + default: false + expiry: 2023/01/01 + owner: ctiller@google.com + test_tags: ["core_end2end_test", "lame_client_test"] diff --git a/src/core/lib/iomgr/closure.h b/src/core/lib/iomgr/closure.h index e607cf5bbd0..095021b2e63 100644 --- a/src/core/lib/iomgr/closure.h +++ b/src/core/lib/iomgr/closure.h @@ -118,6 +118,54 @@ inline grpc_closure* grpc_closure_init(grpc_closure* closure, grpc_closure_init(closure, cb, cb_arg) #endif +namespace grpc_core { +template +grpc_closure MakeMemberClosure(T* p, DebugLocation location = DebugLocation()) { + grpc_closure out; + GRPC_CLOSURE_INIT( + &out, [](void* p, grpc_error_handle e) { (static_cast(p)->*cb)(e); }, + p, nullptr); +#ifndef NDEBUG + out.file_created = location.file(); + out.line_created = location.line(); +#else + (void)location; +#endif + return out; +} + +template +grpc_closure MakeMemberClosure(T* p, DebugLocation location = DebugLocation()) { + grpc_closure out; + GRPC_CLOSURE_INIT( + &out, [](void* p, grpc_error_handle) { (static_cast(p)->*cb)(); }, p, + nullptr); +#ifndef NDEBUG + out.file_created = location.file(); + out.line_created = location.line(); +#else + (void)location; +#endif + return out; +} + +template +grpc_closure* NewClosure(F f) { + struct Closure : public grpc_closure { + explicit Closure(F f) : f(std::move(f)) {} + F f; + static void Run(void* arg, grpc_error_handle error) { + auto self = static_cast(arg); + self->f(error); + delete self; + } + }; + Closure* c = new Closure(std::move(f)); + GRPC_CLOSURE_INIT(c, Closure::Run, c, nullptr); + return c; +} +} // namespace grpc_core + namespace closure_impl { struct wrapped_closure { diff --git a/src/core/lib/iomgr/socket_utils_posix.h b/src/core/lib/iomgr/socket_utils_posix.h index 27c3b284edb..213d5c33160 100644 --- a/src/core/lib/iomgr/socket_utils_posix.h +++ b/src/core/lib/iomgr/socket_utils_posix.h @@ -64,7 +64,7 @@ struct PosixTcpOptions { PosixTcpOptions() = default; // Move ctor PosixTcpOptions(PosixTcpOptions&& other) noexcept { - socket_mutator = absl::exchange(other.socket_mutator, nullptr); + socket_mutator = std::exchange(other.socket_mutator, nullptr); resource_quota = std::move(other.resource_quota); CopyIntegerOptions(other); } @@ -73,7 +73,7 @@ struct PosixTcpOptions { if (socket_mutator != nullptr) { grpc_socket_mutator_unref(socket_mutator); } - socket_mutator = absl::exchange(other.socket_mutator, nullptr); + socket_mutator = std::exchange(other.socket_mutator, nullptr); resource_quota = std::move(other.resource_quota); CopyIntegerOptions(other); return *this; diff --git a/src/core/lib/promise/activity.cc b/src/core/lib/promise/activity.cc index 7c2ed56e4c6..c9304efcc1e 100644 --- a/src/core/lib/promise/activity.cc +++ b/src/core/lib/promise/activity.cc @@ -18,6 +18,8 @@ #include +#include "absl/strings/str_format.h" + #include "src/core/lib/gprpp/atomic_utils.h" namespace grpc_core { @@ -32,6 +34,8 @@ namespace promise_detail { /////////////////////////////////////////////////////////////////////////////// // HELPER TYPES +std::string Unwakeable::ActivityDebugTag() const { return ""; } + // Weak handle to an Activity. // Handle can persist while Activity goes away. class FreestandingActivity::Handle final : public Wakeable { @@ -74,6 +78,11 @@ class FreestandingActivity::Handle final : public Wakeable { void Drop() override { Unref(); } + std::string ActivityDebugTag() const override { + MutexLock lock(&mu_); + return activity_ == nullptr ? "" : activity_->DebugTag(); + } + private: // Unref the Handle (not the activity). void Unref() { @@ -85,7 +94,7 @@ class FreestandingActivity::Handle final : public Wakeable { // Two initial refs: one for the waiter that caused instantiation, one for the // activity. std::atomic refs_{2}; - Mutex mu_ ABSL_ACQUIRED_AFTER(activity_->mu_); + mutable Mutex mu_ ABSL_ACQUIRED_AFTER(activity_->mu_); FreestandingActivity* activity_ ABSL_GUARDED_BY(mu_); }; @@ -117,4 +126,9 @@ Waker FreestandingActivity::MakeNonOwningWaker() { } } // namespace promise_detail + +std::string Activity::DebugTag() const { + return absl::StrFormat("ACTIVITY[%p]", this); +} + } // namespace grpc_core diff --git a/src/core/lib/promise/activity.h b/src/core/lib/promise/activity.h index d89d5144194..090f89a93f5 100644 --- a/src/core/lib/promise/activity.h +++ b/src/core/lib/promise/activity.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "absl/base/thread_annotations.h" @@ -43,6 +44,8 @@ namespace grpc_core { +class Activity; + // A Wakeable object is used by queues to wake activities. class Wakeable { public: @@ -52,19 +55,23 @@ class Wakeable { // Drop this wakeable without waking up the underlying activity. virtual void Drop() = 0; + // Return the underlying activity debug tag, or "" if not available. + virtual std::string ActivityDebugTag() const = 0; + protected: inline ~Wakeable() {} }; -namespace activity_detail { +namespace promise_detail { struct Unwakeable final : public Wakeable { void Wakeup() override {} void Drop() override {} + std::string ActivityDebugTag() const override; }; static Unwakeable* unwakeable() { return NoDestructSingleton::Get(); } -} // namespace activity_detail +} // namespace promise_detail class AtomicWaker; @@ -73,7 +80,7 @@ class AtomicWaker; class Waker { public: explicit Waker(Wakeable* wakeable) : wakeable_(wakeable) {} - Waker() : Waker(activity_detail::unwakeable()) {} + Waker() : Waker(promise_detail::unwakeable()) {} ~Waker() { wakeable_->Drop(); } Waker(const Waker&) = delete; Waker& operator=(const Waker&) = delete; @@ -95,11 +102,15 @@ class Waker { return wakeable_ == other.wakeable_; } + std::string ActivityDebugTag() { + return wakeable_ == nullptr ? "" : wakeable_->ActivityDebugTag(); + } + private: friend class AtomicWaker; Wakeable* Take() { - return std::exchange(wakeable_, activity_detail::unwakeable()); + return std::exchange(wakeable_, promise_detail::unwakeable()); } Wakeable* wakeable_; @@ -109,7 +120,7 @@ class Waker { class AtomicWaker { public: explicit AtomicWaker(Wakeable* wakeable) : wakeable_(wakeable) {} - AtomicWaker() : AtomicWaker(activity_detail::unwakeable()) {} + AtomicWaker() : AtomicWaker(promise_detail::unwakeable()) {} explicit AtomicWaker(Waker waker) : AtomicWaker(waker.Take()) {} ~AtomicWaker() { wakeable_.load(std::memory_order_acquire)->Drop(); } AtomicWaker(const AtomicWaker&) = delete; @@ -123,7 +134,7 @@ class AtomicWaker { // Return true if there is a not-unwakeable wakeable present. bool Armed() const noexcept { return wakeable_.load(std::memory_order_relaxed) != - activity_detail::unwakeable(); + promise_detail::unwakeable(); } // Set to some new waker @@ -133,7 +144,7 @@ class AtomicWaker { private: Wakeable* Take() { - return wakeable_.exchange(activity_detail::unwakeable(), + return wakeable_.exchange(promise_detail::unwakeable(), std::memory_order_acq_rel); } @@ -179,6 +190,9 @@ class Activity : public Orphanable { // delivered until long after the activity should be destroyed. virtual Waker MakeNonOwningWaker() = 0; + // Some descriptive text to add to log messages to identify this activity. + virtual std::string DebugTag() const; + protected: // Check if this activity is the current activity executing on the current // thread. @@ -333,6 +347,8 @@ class FreestandingActivity : public Activity, private Wakeable { Mutex* mu() ABSL_LOCK_RETURNED(mu_) { return &mu_; } + std::string ActivityDebugTag() const override { return DebugTag(); } + private: class Handle; diff --git a/src/core/lib/promise/call_push_pull.h b/src/core/lib/promise/call_push_pull.h index 9c68f259115..9f39d7ebb42 100644 --- a/src/core/lib/promise/call_push_pull.h +++ b/src/core/lib/promise/call_push_pull.h @@ -86,13 +86,9 @@ class CallPushPull { if (!done_.is_set(kDoneMain)) { auto p = main_(); if (auto* status = absl::get_if(&p)) { - if (IsStatusOk(*status)) { - done_.set(kDoneMain); - Destruct(&main_); - Construct(&result_, std::move(*status)); - } else { - return std::move(*status); - } + done_.set(kDoneMain); + Destruct(&main_); + Construct(&result_, std::move(*status)); } } if (!done_.is_set(kDonePull)) { @@ -134,6 +130,10 @@ class CallPushPull { // When polling, the push is polled first, then the main call (descending the // stack), then the pull (as we ascend once more). // +// If the push or the pull fail early, then the entire call fails. +// If the main part of the call fails, we wait until both push and pull are also +// done. +// // This strategy minimizes repolls. template promise_detail::CallPushPull CallPushPull(FMain f_main, diff --git a/src/core/lib/promise/pipe.h b/src/core/lib/promise/pipe.h index e87cf25cba1..d54e4b9ac9a 100644 --- a/src/core/lib/promise/pipe.h +++ b/src/core/lib/promise/pipe.h @@ -240,6 +240,8 @@ class Center { template class PipeSender { public: + using PushType = pipe_detail::Push; + PipeSender(const PipeSender&) = delete; PipeSender& operator=(const PipeSender&) = delete; @@ -257,11 +259,15 @@ class PipeSender { if (center_ != nullptr) center_->UnrefSend(); } + void Close() { + if (auto* center = std::exchange(center_, nullptr)) center->UnrefSend(); + } + // Send a single message along the pipe. // Returns a promise that will resolve to a bool - true if the message was // sent, false if it could never be sent. Blocks the promise until the // receiver is either closed or able to receive another message. - pipe_detail::Push Push(T value); + PushType Push(T value); private: friend struct Pipe; @@ -273,6 +279,8 @@ class PipeSender { template class PipeReceiver { public: + using NextType = pipe_detail::Next; + PipeReceiver(const PipeReceiver&) = delete; PipeReceiver& operator=(const PipeReceiver&) = delete; @@ -294,7 +302,7 @@ class PipeReceiver { // message was received, or no value if the other end of the pipe was closed. // Blocks the promise until the receiver is either closed or a message is // available. - pipe_detail::Next Next(); + NextType Next(); private: friend struct Pipe; @@ -434,7 +442,8 @@ void NextResult::reset() { // polling code) would likely be more appropriate. template struct Pipe { - Pipe() : Pipe(GetContext()->New>()) {} + Pipe() : Pipe(GetContext()) {} + explicit Pipe(Arena* arena) : Pipe(arena->New>()) {} Pipe(const Pipe&) = delete; Pipe& operator=(const Pipe&) = delete; Pipe(Pipe&&) noexcept = default; diff --git a/src/core/lib/promise/poll.h b/src/core/lib/promise/poll.h index e891bf728e9..23cea7ea09a 100644 --- a/src/core/lib/promise/poll.h +++ b/src/core/lib/promise/poll.h @@ -19,6 +19,8 @@ #include +#include + #include "absl/types/variant.h" namespace grpc_core { @@ -61,6 +63,17 @@ struct PollTraits> { static constexpr bool is_poll() { return true; } }; +// Convert a poll to a string +template +std::string PollToString( + const Poll& poll, + F t_to_string = [](const T& t) { return t.ToString(); }) { + if (absl::holds_alternative(poll)) { + return "<>"; + } + return t_to_string(absl::get(poll)); +} + } // namespace grpc_core #endif // GRPC_CORE_LIB_PROMISE_POLL_H diff --git a/src/core/lib/security/authorization/grpc_server_authz_filter.cc b/src/core/lib/security/authorization/grpc_server_authz_filter.cc index 3139eadf287..4ee98da9a1a 100644 --- a/src/core/lib/security/authorization/grpc_server_authz_filter.cc +++ b/src/core/lib/security/authorization/grpc_server_authz_filter.cc @@ -32,6 +32,7 @@ #include "src/core/lib/promise/promise.h" #include "src/core/lib/security/authorization/authorization_engine.h" #include "src/core/lib/security/authorization/evaluate_args.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { diff --git a/src/core/lib/security/authorization/grpc_server_authz_filter.h b/src/core/lib/security/authorization/grpc_server_authz_filter.h index b324513ec90..d210df59ccb 100644 --- a/src/core/lib/security/authorization/grpc_server_authz_filter.h +++ b/src/core/lib/security/authorization/grpc_server_authz_filter.h @@ -30,6 +30,7 @@ #include "src/core/lib/security/authorization/authorization_policy_provider.h" #include "src/core/lib/security/authorization/evaluate_args.h" #include "src/core/lib/security/context/security_context.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/transport.h" namespace grpc_core { diff --git a/src/core/lib/security/credentials/call_creds_util.h b/src/core/lib/security/credentials/call_creds_util.h index 75b6e83c5f6..cb66c1db069 100644 --- a/src/core/lib/security/credentials/call_creds_util.h +++ b/src/core/lib/security/credentials/call_creds_util.h @@ -24,7 +24,7 @@ #include #include "src/core/lib/security/credentials/credentials.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/call_fragments.h" namespace grpc_core { diff --git a/src/core/lib/security/credentials/composite/composite_credentials.cc b/src/core/lib/security/credentials/composite/composite_credentials.cc index 62faee5cef4..abb2412f960 100644 --- a/src/core/lib/security/credentials/composite/composite_credentials.cc +++ b/src/core/lib/security/credentials/composite/composite_credentials.cc @@ -33,7 +33,7 @@ #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/promise/try_seq.h" #include "src/core/lib/surface/api_trace.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/call_fragments.h" // // grpc_composite_channel_credentials diff --git a/src/core/lib/security/credentials/composite/composite_credentials.h b/src/core/lib/security/credentials/composite/composite_credentials.h index 045216e9a4a..ad432816948 100644 --- a/src/core/lib/security/credentials/composite/composite_credentials.h +++ b/src/core/lib/security/credentials/composite/composite_credentials.h @@ -39,7 +39,7 @@ #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/security_connector/security_connector.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/call_fragments.h" /* -- Composite channel credentials. -- */ diff --git a/src/core/lib/security/credentials/fake/fake_credentials.cc b/src/core/lib/security/credentials/fake/fake_credentials.cc index 94056afc4b9..760a296934d 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.cc +++ b/src/core/lib/security/credentials/fake/fake_credentials.cc @@ -31,6 +31,7 @@ #include "src/core/lib/promise/promise.h" #include "src/core/lib/security/security_connector/fake/fake_security_connector.h" #include "src/core/lib/security/security_connector/security_connector.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" /* -- Fake transport security credentials. -- */ diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h index 4e8e8556a74..4226e0aec3a 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.h +++ b/src/core/lib/security/credentials/fake/fake_credentials.h @@ -35,7 +35,7 @@ #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/call_fragments.h" #define GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS \ "grpc.fake_security.expected_targets" diff --git a/src/core/lib/security/credentials/iam/iam_credentials.cc b/src/core/lib/security/credentials/iam/iam_credentials.cc index 95fb93b5be3..0138f618331 100644 --- a/src/core/lib/security/credentials/iam/iam_credentials.cc +++ b/src/core/lib/security/credentials/iam/iam_credentials.cc @@ -34,6 +34,7 @@ #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/promise/promise.h" #include "src/core/lib/surface/api_trace.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" grpc_core::ArenaPromise> diff --git a/src/core/lib/security/credentials/iam/iam_credentials.h b/src/core/lib/security/credentials/iam/iam_credentials.h index f0c83af549e..e01a86ffeb1 100644 --- a/src/core/lib/security/credentials/iam/iam_credentials.h +++ b/src/core/lib/security/credentials/iam/iam_credentials.h @@ -33,7 +33,7 @@ #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/call_fragments.h" class grpc_google_iam_credentials : public grpc_call_credentials { public: diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.cc b/src/core/lib/security/credentials/jwt/jwt_credentials.cc index 94061b24a69..798fe8cc7b0 100644 --- a/src/core/lib/security/credentials/jwt/jwt_credentials.cc +++ b/src/core/lib/security/credentials/jwt/jwt_credentials.cc @@ -42,6 +42,7 @@ #include "src/core/lib/promise/promise.h" #include "src/core/lib/security/credentials/call_creds_util.h" #include "src/core/lib/surface/api_trace.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/uri/uri_parser.h" diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.h b/src/core/lib/security/credentials/jwt/jwt_credentials.h index 226a95a6577..4c51d892f14 100644 --- a/src/core/lib/security/credentials/jwt/jwt_credentials.h +++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h @@ -43,7 +43,7 @@ #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/credentials/jwt/json_token.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/call_fragments.h" class grpc_service_account_jwt_access_credentials : public grpc_call_credentials { diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc b/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc index 78a4f509773..e3d53c4d61f 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc @@ -57,9 +57,9 @@ #include "src/core/lib/promise/promise.h" #include "src/core/lib/security/util/json_util.h" #include "src/core/lib/surface/api_trace.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" -#include "src/core/lib/transport/transport.h" #include "src/core/lib/uri/uri_parser.h" using grpc_core::Json; diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h index 5ecfd80d89b..33d20048a51 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h @@ -48,7 +48,7 @@ #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/uri/uri_parser.h" // Constants. diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.cc b/src/core/lib/security/credentials/plugin/plugin_credentials.cc index cd832d77178..c5b32ab50c8 100644 --- a/src/core/lib/security/credentials/plugin/plugin_credentials.cc +++ b/src/core/lib/security/credentials/plugin/plugin_credentials.cc @@ -37,6 +37,7 @@ #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/validate_metadata.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" grpc_core::TraceFlag grpc_plugin_credentials_trace(false, "plugin_credentials"); diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h index a8cd4f5baa2..07148d042a6 100644 --- a/src/core/lib/security/credentials/plugin/plugin_credentials.h +++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h @@ -46,7 +46,7 @@ #include "src/core/lib/security/credentials/call_creds_util.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/slice/slice.h" -#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/call_fragments.h" extern grpc_core::TraceFlag grpc_plugin_credentials_trace; diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc index 11117fc4a8a..223dc3364a8 100644 --- a/src/core/lib/security/transport/client_auth_filter.cc +++ b/src/core/lib/security/transport/client_auth_filter.cc @@ -52,6 +52,7 @@ #include "src/core/lib/security/security_connector/security_connector.h" #include "src/core/lib/security/transport/auth_filters.h" #include "src/core/lib/slice/slice.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 954ab54cb3f..f20d2c6a1b3 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -20,24 +20,32 @@ #include "src/core/lib/surface/call.h" +#include #include #include #include #include +#include #include #include #include +#include #include "absl/base/thread_annotations.h" +#include "absl/cleanup/cleanup.h" +#include "absl/container/inlined_vector.h" #include "absl/meta/type_traits.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" #include "absl/strings/string_view.h" +#include "absl/types/variant.h" #include #include +#include #include #include #include @@ -49,20 +57,32 @@ #include #include +#include "src/core/lib/channel/call_finalization.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/channelz.h" #include "src/core/lib/channel/context.h" +#include "src/core/lib/channel/status_util.h" #include "src/core/lib/compression/compression_internal.h" #include "src/core/lib/debug/stats.h" +#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gpr/alloc.h" #include "src/core/lib/gpr/time_precise.h" +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/bitset.h" #include "src/core/lib/gprpp/cpp_impl_of.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/call_combiner.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/polling_entity.h" +#include "src/core/lib/promise/activity.h" +#include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/promise/context.h" +#include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/pipe.h" +#include "src/core/lib/promise/poll.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice_buffer.h" #include "src/core/lib/slice/slice_internal.h" @@ -72,15 +92,21 @@ #include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/server.h" #include "src/core/lib/surface/validate_metadata.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/error_utils.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" grpc_core::TraceFlag grpc_call_error_trace(false, "call_error"); grpc_core::TraceFlag grpc_compression_trace(false, "compression"); +grpc_core::TraceFlag grpc_call_trace(false, "call"); +grpc_core::TraceFlag grpc_call_refcount_trace(false, "call_refcount"); namespace grpc_core { +/////////////////////////////////////////////////////////////////////////////// +// Call + class Call : public CppImplOf { public: Arena* arena() { return arena_; } @@ -93,7 +119,7 @@ class Call : public CppImplOf { void CancelWithStatus(grpc_status_code status, const char* description); virtual void CancelWithError(grpc_error_handle error) = 0; virtual void SetCompletionQueue(grpc_completion_queue* cq) = 0; - virtual char* GetPeer() = 0; + char* GetPeer(); virtual grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag, bool is_notify_tag_closure) = 0; @@ -115,12 +141,19 @@ class Call : public CppImplOf { // for that functionality be invented) virtual grpc_call_stack* call_stack() = 0; + gpr_atm* peer_string_atm_ptr() { return &peer_string_; } + protected: - Call(Arena* arena, bool is_client, Timestamp send_deadline) - : arena_(arena), send_deadline_(send_deadline), is_client_(is_client) { - GPR_DEBUG_ASSERT(arena_ != nullptr); - } - ~Call() = default; + // The maximum number of concurrent batches possible. + // Based upon the maximum number of individually queueable ops in the batch + // api: + // - initial metadata send + // - message send + // - status/close send (depending on client/server) + // - initial metadata recv + // - message recv + // - status/close recv (depending on client/server) + static constexpr size_t kMaxConcurrentBatches = 6; struct ParentCall { Mutex child_list_mu; @@ -137,8 +170,25 @@ class Call : public CppImplOf { Call* sibling_prev = nullptr; }; + Call(Arena* arena, bool is_client, Timestamp send_deadline, + RefCountedPtr channel) + : channel_(std::move(channel)), + arena_(arena), + send_deadline_(send_deadline), + is_client_(is_client) { + GPR_DEBUG_ASSERT(arena_ != nullptr); + GPR_DEBUG_ASSERT(channel_ != nullptr); + } + virtual ~Call() = default; + + void DeleteThis(); + ParentCall* GetOrCreateParentCall(); ParentCall* parent_call(); + Channel* channel() { + GPR_DEBUG_ASSERT(channel_ != nullptr); + return channel_.get(); + } absl::Status InitParent(Call* parent, uint32_t propagation_mask); void PublishToParent(Call* parent); @@ -150,7 +200,10 @@ class Call : public CppImplOf { send_deadline_ = send_deadline; } + void ClearPeerString() { gpr_atm_rel_store(&peer_string_, 0); } + private: + RefCountedPtr channel_; Arena* const arena_; std::atomic parent_call_{nullptr}; ChildCall* child_ = nullptr; @@ -158,11 +211,150 @@ class Call : public CppImplOf { const bool is_client_; // flag indicating that cancellation is inherited bool cancellation_is_inherited_ = false; + // A char* indicating the peer name. + gpr_atm peer_string_ = 0; }; +Call::ParentCall* Call::GetOrCreateParentCall() { + ParentCall* p = parent_call_.load(std::memory_order_acquire); + if (p == nullptr) { + p = arena_->New(); + ParentCall* expected = nullptr; + if (!parent_call_.compare_exchange_strong(expected, p, + std::memory_order_release, + std::memory_order_relaxed)) { + p->~ParentCall(); + p = expected; + } + } + return p; +} + +Call::ParentCall* Call::parent_call() { + return parent_call_.load(std::memory_order_acquire); +} + +absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) { + child_ = arena()->New(parent); + + parent->InternalRef("child"); + GPR_ASSERT(is_client_); + GPR_ASSERT(!parent->is_client_); + + if (propagation_mask & GRPC_PROPAGATE_DEADLINE) { + send_deadline_ = std::min(send_deadline_, parent->send_deadline_); + } + /* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with + * GRPC_PROPAGATE_STATS_CONTEXT */ + /* TODO(ctiller): This should change to use the appropriate census start_op + * call. */ + if (propagation_mask & GRPC_PROPAGATE_CENSUS_TRACING_CONTEXT) { + if (0 == (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT)) { + return absl::UnknownError( + "Census tracing propagation requested without Census context " + "propagation"); + } + ContextSet(GRPC_CONTEXT_TRACING, parent->ContextGet(GRPC_CONTEXT_TRACING), + nullptr); + } else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) { + return absl::UnknownError( + "Census context propagation requested without Census tracing " + "propagation"); + } + if (propagation_mask & GRPC_PROPAGATE_CANCELLATION) { + cancellation_is_inherited_ = true; + } + return absl::OkStatus(); +} + +void Call::PublishToParent(Call* parent) { + ChildCall* cc = child_; + ParentCall* pc = parent->GetOrCreateParentCall(); + MutexLock lock(&pc->child_list_mu); + if (pc->first_child == nullptr) { + pc->first_child = this; + cc->sibling_next = cc->sibling_prev = this; + } else { + cc->sibling_next = pc->first_child; + cc->sibling_prev = pc->first_child->child_->sibling_prev; + cc->sibling_next->child_->sibling_prev = + cc->sibling_prev->child_->sibling_next = this; + } + if (parent->Completed()) { + CancelWithError(absl::CancelledError()); + } +} + +void Call::MaybeUnpublishFromParent() { + ChildCall* cc = child_; + if (cc == nullptr) return; + + ParentCall* pc = cc->parent->parent_call(); + { + MutexLock lock(&pc->child_list_mu); + if (this == pc->first_child) { + pc->first_child = cc->sibling_next; + if (this == pc->first_child) { + pc->first_child = nullptr; + } + } + cc->sibling_prev->child_->sibling_next = cc->sibling_next; + cc->sibling_next->child_->sibling_prev = cc->sibling_prev; + } + cc->parent->InternalUnref("child"); +} + +void Call::CancelWithStatus(grpc_status_code status, const char* description) { + // copying 'description' is needed to ensure the grpc_call_cancel_with_status + // guarantee that can be short-lived. + CancelWithError(grpc_error_set_int( + grpc_error_set_str(GRPC_ERROR_CREATE_FROM_COPIED_STRING(description), + GRPC_ERROR_STR_GRPC_MESSAGE, description), + GRPC_ERROR_INT_GRPC_STATUS, status)); +} + +void Call::PropagateCancellationToChildren() { + ParentCall* pc = parent_call(); + if (pc != nullptr) { + Call* child; + MutexLock lock(&pc->child_list_mu); + child = pc->first_child; + if (child != nullptr) { + do { + Call* next_child_call = child->child_->sibling_next; + if (child->cancellation_is_inherited_) { + child->InternalRef("propagate_cancel"); + child->CancelWithError(GRPC_ERROR_CANCELLED); + child->InternalUnref("propagate_cancel"); + } + child = next_child_call; + } while (child != pc->first_child); + } + } +} + +char* Call::GetPeer() { + char* peer_string = reinterpret_cast(gpr_atm_acq_load(&peer_string_)); + if (peer_string != nullptr) return gpr_strdup(peer_string); + peer_string = grpc_channel_get_target(channel_->c_ptr()); + if (peer_string != nullptr) return peer_string; + return gpr_strdup("unknown"); +} + +void Call::DeleteThis() { + RefCountedPtr channel = std::move(channel_); + Arena* arena = arena_; + this->~Call(); + channel->UpdateCallSizeEstimate(arena->Destroy()); +} + +/////////////////////////////////////////////////////////////////////////////// +// FilterStackCall +// To be removed once promise conversion is complete + class FilterStackCall final : public Call { public: - ~FilterStackCall() { + ~FilterStackCall() override { for (int i = 0; i < GRPC_CONTEXT_COUNT; ++i) { if (context_[i].destroy) { context_[i].destroy(context_[i].value); @@ -197,7 +389,6 @@ class FilterStackCall final : public Call { void CancelWithError(grpc_error_handle error) override; void SetCompletionQueue(grpc_completion_queue* cq) override; - char* GetPeer() override; grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag, bool is_notify_tag_closure) override; void ExternalRef() override { ext_ref_.Ref(); } @@ -255,17 +446,6 @@ class FilterStackCall final : public Call { } private: - // The maximum number of concurrent batches possible. - // Based upon the maximum number of individually queueable ops in the batch - // api: - // - initial metadata send - // - message send - // - status/close send (depending on client/server) - // - initial metadata recv - // - message recv - // - status/close recv (depending on client/server) - static constexpr size_t kMaxConcurrentBatches = 6; - static constexpr gpr_atm kRecvNone = 0; static constexpr gpr_atm kRecvInitialMetadataFirst = 1; @@ -313,9 +493,9 @@ class FilterStackCall final : public Call { }; FilterStackCall(Arena* arena, const grpc_call_create_args& args) - : Call(arena, args.server_transport_data == nullptr, args.send_deadline), + : Call(arena, args.server_transport_data == nullptr, args.send_deadline, + args.channel->Ref()), cq_(args.cq), - channel_(args.channel->Ref()), stream_op_payload_(context_) {} static void ReleaseCall(void* call, grpc_error_handle); @@ -346,7 +526,6 @@ class FilterStackCall final : public Call { CallCombiner call_combiner_; grpc_completion_queue* cq_; grpc_polling_entity pollent_; - RefCountedPtr channel_; gpr_cycle_counter start_time_ = gpr_get_cycle_counter(); /** has grpc_call_unref been called */ @@ -375,9 +554,6 @@ class FilterStackCall final : public Call { Element 0 is initial metadata, element 1 is trailing metadata. */ grpc_metadata_array* buffered_metadata_[2] = {}; - // A char* indicating the peer name. - gpr_atm peer_string_ = 0; - /* Call data useful used for reporting. Only valid after the call has * completed */ grpc_call_final_info final_info_; @@ -442,76 +618,6 @@ class FilterStackCall final : public Call { gpr_atm recv_state_ = 0; }; -Call::ParentCall* Call::GetOrCreateParentCall() { - ParentCall* p = parent_call_.load(std::memory_order_acquire); - if (p == nullptr) { - p = arena_->New(); - ParentCall* expected = nullptr; - if (!parent_call_.compare_exchange_strong(expected, p, - std::memory_order_release, - std::memory_order_relaxed)) { - p->~ParentCall(); - p = expected; - } - } - return p; -} - -Call::ParentCall* Call::parent_call() { - return parent_call_.load(std::memory_order_acquire); -} - -absl::Status Call::InitParent(Call* parent, uint32_t propagation_mask) { - child_ = arena()->New(parent); - - parent->InternalRef("child"); - GPR_ASSERT(is_client_); - GPR_ASSERT(!parent->is_client_); - - if (propagation_mask & GRPC_PROPAGATE_DEADLINE) { - send_deadline_ = std::min(send_deadline_, parent->send_deadline_); - } - /* for now GRPC_PROPAGATE_TRACING_CONTEXT *MUST* be passed with - * GRPC_PROPAGATE_STATS_CONTEXT */ - /* TODO(ctiller): This should change to use the appropriate census start_op - * call. */ - if (propagation_mask & GRPC_PROPAGATE_CENSUS_TRACING_CONTEXT) { - if (0 == (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT)) { - return absl::UnknownError( - "Census tracing propagation requested without Census context " - "propagation"); - } - ContextSet(GRPC_CONTEXT_TRACING, parent->ContextGet(GRPC_CONTEXT_TRACING), - nullptr); - } else if (propagation_mask & GRPC_PROPAGATE_CENSUS_STATS_CONTEXT) { - return absl::UnknownError( - "Census context propagation requested without Census tracing " - "propagation"); - } - if (propagation_mask & GRPC_PROPAGATE_CANCELLATION) { - cancellation_is_inherited_ = true; - } - return absl::OkStatus(); -} - -void Call::PublishToParent(Call* parent) { - ChildCall* cc = child_; - ParentCall* pc = parent->GetOrCreateParentCall(); - MutexLock lock(&pc->child_list_mu); - if (pc->first_child == nullptr) { - pc->first_child = this; - cc->sibling_next = cc->sibling_prev = this; - } else { - cc->sibling_next = pc->first_child; - cc->sibling_prev = pc->first_child->child_->sibling_prev; - cc->sibling_next->child_->sibling_prev = - cc->sibling_prev->child_->sibling_next = this; - } - if (parent->Completed()) { - CancelWithError(absl::CancelledError()); - } -} - grpc_error_handle FilterStackCall::Create(grpc_call_create_args* args, grpc_call** out_call) { Channel* channel = args->channel.get(); @@ -631,11 +737,7 @@ void FilterStackCall::SetCompletionQueue(grpc_completion_queue* cq) { } void FilterStackCall::ReleaseCall(void* call, grpc_error_handle /*error*/) { - auto* c = static_cast(call); - RefCountedPtr channel = std::move(c->channel_); - Arena* arena = c->arena(); - c->~FilterStackCall(); - channel->UpdateCallSizeEstimate(arena->Destroy()); + static_cast(call)->DeleteThis(); } void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) { @@ -663,27 +765,8 @@ void FilterStackCall::DestroyCall(void* call, grpc_error_handle /*error*/) { grpc_schedule_on_exec_ctx)); } -void Call::MaybeUnpublishFromParent() { - ChildCall* cc = child_; - if (cc == nullptr) return; - - ParentCall* pc = cc->parent->parent_call(); - { - MutexLock lock(&pc->child_list_mu); - if (this == pc->first_child) { - pc->first_child = cc->sibling_next; - if (this == pc->first_child) { - pc->first_child = nullptr; - } - } - cc->sibling_prev->child_->sibling_next = cc->sibling_next; - cc->sibling_next->child_->sibling_prev = cc->sibling_prev; - } - cc->parent->InternalUnref("child"); -} - -void FilterStackCall::ExternalUnref() { - if (GPR_LIKELY(!ext_ref_.Unref())) return; +void FilterStackCall::ExternalUnref() { + if (GPR_LIKELY(!ext_ref_.Unref())) return; ApplicationCallbackExecCtx callback_exec_ctx; ExecCtx exec_ctx; @@ -707,14 +790,6 @@ void FilterStackCall::ExternalUnref() { InternalUnref("destroy"); } -char* FilterStackCall::GetPeer() { - char* peer_string = reinterpret_cast(gpr_atm_acq_load(&peer_string_)); - if (peer_string != nullptr) return gpr_strdup(peer_string); - peer_string = grpc_channel_get_target(channel_->c_ptr()); - if (peer_string != nullptr) return peer_string; - return gpr_strdup("unknown"); -} - // start_batch_closure points to a caller-allocated closure to be used // for entering the call combiner. void FilterStackCall::ExecuteBatch(grpc_transport_stream_op_batch* batch, @@ -759,7 +834,7 @@ void FilterStackCall::CancelWithError(grpc_error_handle error) { if (!gpr_atm_rel_cas(&cancelled_with_error_, 0, 1)) { return; } - gpr_atm_rel_store(&peer_string_, 0); + ClearPeerString(); InternalRef("termination"); // Inform the call combiner of the cancellation, so that it can cancel // any in-flight asynchronous actions that may be holding the call @@ -777,15 +852,6 @@ void FilterStackCall::CancelWithError(grpc_error_handle error) { ExecuteBatch(op, &state->start_batch); } -void Call::CancelWithStatus(grpc_status_code status, const char* description) { - // copying 'description' is needed to ensure the grpc_call_cancel_with_status - // guarantee that can be short-lived. - CancelWithError(grpc_error_set_int( - grpc_error_set_str(GRPC_ERROR_CREATE_FROM_COPIED_STRING(description), - GRPC_ERROR_STR_GRPC_MESSAGE, description), - GRPC_ERROR_INT_GRPC_STATUS, status)); -} - void FilterStackCall::SetFinalStatus(grpc_error_handle error) { if (GRPC_TRACE_FLAG_ENABLED(grpc_call_error_trace)) { gpr_log(GPR_DEBUG, "set_final_status %s", is_client() ? "CLI" : "SVR"); @@ -799,7 +865,7 @@ void FilterStackCall::SetFinalStatus(grpc_error_handle error) { *final_op_.client.status_details = grpc_slice_from_cpp_string(std::move(status_details)); status_error_.set(error); - channelz::ChannelNode* channelz_channel = channel_->channelz_node(); + channelz::ChannelNode* channelz_channel = channel()->channelz_node(); if (channelz_channel != nullptr) { if (*final_op_.client.status != GRPC_STATUS_OK) { channelz_channel->RecordCallFailed(); @@ -1035,26 +1101,6 @@ FilterStackCall::BatchControl* FilterStackCall::ReuseOrAllocateBatchControl( return bctl; } -void Call::PropagateCancellationToChildren() { - ParentCall* pc = parent_call(); - if (pc != nullptr) { - Call* child; - MutexLock lock(&pc->child_list_mu); - child = pc->first_child; - if (child != nullptr) { - do { - Call* next_child_call = child->child_->sibling_next; - if (child->cancellation_is_inherited_) { - child->InternalRef("propagate_cancel"); - child->CancelWithError(absl::CancelledError()); - child->InternalUnref("propagate_cancel"); - } - child = next_child_call; - } while (child != pc->first_child); - } - } -} - void FilterStackCall::BatchControl::PostCompletion() { FilterStackCall* call = call_; grpc_error_handle error = batch_error_.get(); @@ -1182,7 +1228,7 @@ void FilterStackCall::BatchControl::ValidateFilteredMetadata() { FilterStackCall* call = call_; const grpc_compression_options compression_options = - call->channel_->compression_options(); + call->channel()->compression_options(); const grpc_compression_algorithm compression_algorithm = call->incoming_compression_algorithm_; if (GPR_UNLIKELY(!CompressionAlgorithmSet::FromUint32( @@ -1279,6 +1325,24 @@ void FilterStackCall::BatchControl::FinishBatch(grpc_error_handle error) { FinishStep(); } +namespace { +void EndOpImmediately(grpc_completion_queue* cq, void* notify_tag, + bool is_notify_tag_closure) { + if (!is_notify_tag_closure) { + GPR_ASSERT(grpc_cq_begin_op(cq, notify_tag)); + grpc_cq_end_op( + cq, notify_tag, GRPC_ERROR_NONE, + [](void*, grpc_cq_completion* completion) { gpr_free(completion); }, + nullptr, + static_cast( + gpr_malloc(sizeof(grpc_cq_completion)))); + } else { + Closure::Run(DEBUG_LOCATION, static_cast(notify_tag), + GRPC_ERROR_NONE); + } +} +} // namespace + grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, void* notify_tag, bool is_notify_tag_closure) { @@ -1302,18 +1366,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, GRPC_CALL_LOG_BATCH(GPR_INFO, ops, nops); if (nops == 0) { - if (!is_notify_tag_closure) { - GPR_ASSERT(grpc_cq_begin_op(cq_, notify_tag)); - grpc_cq_end_op( - cq_, notify_tag, absl::OkStatus(), - [](void*, grpc_cq_completion* completion) { gpr_free(completion); }, - nullptr, - static_cast( - gpr_malloc(sizeof(grpc_cq_completion)))); - } else { - Closure::Run(DEBUG_LOCATION, static_cast(notify_tag), - absl::OkStatus()); - } + EndOpImmediately(cq_, notify_tag, is_notify_tag_closure); error = GRPC_CALL_OK; goto done; } @@ -1361,7 +1414,7 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, level_set = true; } else { const grpc_compression_options copts = - channel_->compression_options(); + channel()->compression_options(); if (copts.default_level.is_set) { level_set = true; effective_compression_level = copts.default_level.level; @@ -1406,7 +1459,8 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, stream_op_payload->send_initial_metadata.send_initial_metadata = &send_initial_metadata_; if (is_client()) { - stream_op_payload->send_initial_metadata.peer_string = &peer_string_; + stream_op_payload->send_initial_metadata.peer_string = + peer_string_atm_ptr(); } has_send_ops = true; break; @@ -1558,7 +1612,8 @@ grpc_call_error FilterStackCall::StartBatch(const grpc_op* ops, size_t nops, stream_op_payload->recv_initial_metadata.trailing_metadata_available = &is_trailers_only_; } else { - stream_op_payload->recv_initial_metadata.peer_string = &peer_string_; + stream_op_payload->recv_initial_metadata.peer_string = + peer_string_atm_ptr(); } ++num_recv_ops; break; @@ -1729,8 +1784,1090 @@ void FilterStackCall::ContextSet(grpc_context_index elem, void* value, context_[elem].destroy = destroy; } +/////////////////////////////////////////////////////////////////////////////// +// Metadata validation helpers + +namespace { +bool ValidateMetadata(size_t count, grpc_metadata* metadata) { + for (size_t i = 0; i < count; i++) { + grpc_metadata* md = &metadata[i]; + if (!GRPC_LOG_IF_ERROR("validate_metadata", + grpc_validate_header_key_is_legal(md->key))) { + return false; + } else if (!grpc_is_binary_header_internal(md->key) && + !GRPC_LOG_IF_ERROR( + "validate_metadata", + grpc_validate_header_nonbin_value_is_legal(md->value))) { + return false; + } else if (GRPC_SLICE_LENGTH(md->value) >= UINT32_MAX) { + // HTTP2 hpack encoding has a maximum limit. + return false; + } + } + return true; +} +} // namespace + +/////////////////////////////////////////////////////////////////////////////// +// PromiseBasedCall +// Will be folded into Call once the promise conversion is done + +class PromiseBasedCall : public Call, public Activity, public Wakeable { + public: + PromiseBasedCall(Arena* arena, const grpc_call_create_args& args); + + void ContextSet(grpc_context_index elem, void* value, + void (*destroy)(void* value)) override; + void* ContextGet(grpc_context_index elem) const override; + void SetCompletionQueue(grpc_completion_queue* cq) override; + + // Implementation of call refcounting: move this to DualRefCounted once we + // don't need to maintain FilterStackCall compatibility + void ExternalRef() final { + refs_.fetch_add(MakeRefPair(1, 0), std::memory_order_relaxed); + } + void ExternalUnref() final { + const uint64_t prev_ref_pair = + refs_.fetch_add(MakeRefPair(-1, 1), std::memory_order_acq_rel); + const uint32_t strong_refs = GetStrongRefs(prev_ref_pair); + if (GPR_UNLIKELY(strong_refs == 1)) { + Orphan(); + } + // Now drop the weak ref. + InternalUnref("external_ref"); + } + void InternalRef(const char*) final { + refs_.fetch_add(MakeRefPair(0, 1), std::memory_order_relaxed); + } + void InternalUnref(const char*) final { + const uint64_t prev_ref_pair = + refs_.fetch_sub(MakeRefPair(0, 1), std::memory_order_acq_rel); + if (GPR_UNLIKELY(prev_ref_pair == MakeRefPair(0, 1))) { + DeleteThis(); + } + } + + // Activity methods + void ForceImmediateRepoll() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override; + Waker MakeOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override { + InternalRef("wakeup"); +// If ASAN is defined, we leverage it to detect dropped Waker objects. +// Usually Waker must be destroyed or woken up, but (especially with arenas) +// it's not uncommon to create a Waker and then do neither. In that case it's +// incredibly fraught to diagnose where the dropped reference to this object was +// created. Instead, leverage ASAN and create a new object per expected wakeup. +// Now when we drop such an object ASAN will fail and we'll get a callstack to +// the creation of the waker in question. +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define GRPC_CALL_USES_ASAN_WAKER + class AsanWaker final : public Wakeable { + public: + explicit AsanWaker(PromiseBasedCall* call) : call_(call) {} + + void Wakeup() override { + call_->Wakeup(); + delete this; + } + + void Drop() override { + call_->Drop(); + delete this; + } + + std::string ActivityDebugTag() const override { + return call_->DebugTag(); + } + + private: + PromiseBasedCall* call_; + }; + return Waker(new AsanWaker(this)); +#endif +#endif +#ifndef GRPC_CALL_USES_ASAN_WAKER + return Waker(this); +#endif + } + Waker MakeNonOwningWaker() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) override; + + // Wakeable methods + void Wakeup() override { + channel()->event_engine()->Run([this] { + ApplicationCallbackExecCtx app_exec_ctx; + ExecCtx exec_ctx; + { + ScopedContext activity_context(this); + MutexLock lock(&mu_); + Update(); + } + InternalUnref("wakeup"); + }); + } + void Drop() override { InternalUnref("wakeup"); } + + void RunInContext(absl::AnyInvocable fn) { + if (Activity::current() == this) { + fn(); + } else { + InternalRef("in_context"); + channel()->event_engine()->Run([this, fn = std::move(fn)]() mutable { + ApplicationCallbackExecCtx app_exec_ctx; + ExecCtx exec_ctx; + { + ScopedContext activity_context(this); + MutexLock lock(&mu_); + fn(); + Update(); + } + InternalUnref("in_context"); + }); + } + } + + grpc_compression_algorithm test_only_compression_algorithm() override { + abort(); + } + uint32_t test_only_message_flags() override { abort(); } + uint32_t test_only_encodings_accepted_by_peer() override { abort(); } + grpc_compression_algorithm compression_for_level( + grpc_compression_level) override { + abort(); + } + + // This should return nullptr for the promise stack (and alternative means + // for that functionality be invented) + grpc_call_stack* call_stack() override { return nullptr; } + + protected: + class ScopedContext + : public ScopedActivity, + public promise_detail::Context, + public promise_detail::Context, + public promise_detail::Context, + public promise_detail::Context, + public promise_detail::Context { + public: + explicit ScopedContext(PromiseBasedCall* call) + : ScopedActivity(call), + promise_detail::Context(call->arena()), + promise_detail::Context(call->context_), + promise_detail::Context(&call->call_context_), + promise_detail::Context(&call->finalization_), + promise_detail::Context( + &call->fragment_allocator_) {} + }; + + class Completion { + public: + Completion() : index_(kNullIndex) {} + ~Completion() { GPR_ASSERT(index_ == kNullIndex); } + explicit Completion(uint8_t index) : index_(index) {} + Completion(const Completion& other) = delete; + Completion& operator=(const Completion& other) = delete; + Completion(Completion&& other) noexcept : index_(other.index_) { + other.index_ = kNullIndex; + } + Completion& operator=(Completion&& other) noexcept { + GPR_ASSERT(index_ == kNullIndex); + index_ = other.index_; + other.index_ = kNullIndex; + return *this; + } + + uint8_t index() const { return index_; } + uint8_t TakeIndex() { return std::exchange(index_, kNullIndex); } + bool has_value() const { return index_ != kNullIndex; } + + std::string ToString() const { + return index_ == kNullIndex ? "null" + : std::to_string(static_cast(index_)); + } + + private: + enum : uint8_t { kNullIndex = 0xff }; + uint8_t index_; + }; + + ~PromiseBasedCall() override { + if (non_owning_wakeable_) non_owning_wakeable_->DropActivity(); + if (cq_) GRPC_CQ_INTERNAL_UNREF(cq_, "bind"); + } + + // Enumerates why a Completion is still pending + enum class PendingOp { + // We're in the midst of starting a batch of operations + kStartingBatch = 0, + // The following correspond with the batch operations from above + kReceiveInitialMetadata, + kReceiveStatusOnClient, + kSendMessage, + kReceiveMessage, + }; + + static constexpr const char* PendingOpString(PendingOp reason) { + switch (reason) { + case PendingOp::kStartingBatch: + return "StartingBatch"; + case PendingOp::kReceiveInitialMetadata: + return "ReceiveInitialMetadata"; + case PendingOp::kReceiveStatusOnClient: + return "ReceiveStatusOnClient"; + case PendingOp::kSendMessage: + return "SendMessage"; + case PendingOp::kReceiveMessage: + return "ReceiveMessage"; + } + return "Unknown"; + } + + static constexpr uint8_t PendingOpBit(PendingOp reason) { + return 1 << static_cast(reason); + } + + Mutex* mu() const ABSL_LOCK_RETURNED(mu_) { return &mu_; } + + // Begin work on a completion, recording the tag/closure to notify. + // Use the op selected in \a ops to determine the index to allocate into. + // Starts the "StartingBatch" PendingOp immediately. + // Assumes at least one operation in \a ops. + Completion StartCompletion(void* tag, bool is_closure, const grpc_op* ops) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Add one pending op to the completion, and return it. + Completion AddOpToCompletion(const Completion& completion, PendingOp reason) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Finish one op on the completion. Must have been previously been added. + // The completion as a whole finishes when all pending ops finish. + void FinishOpOnCompletion(Completion* completion, PendingOp reason) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Mark the completion as failed. Does not finish it. + void FailCompletion(const Completion& completion); + // Run the promise polling loop until it stalls. + void Update() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Update the promise state once. + virtual void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0; + // Accept the stats from the context (call once we have proof the transport is + // done with them). + // Right now this means that promise based calls do not record correct stats + // with census if they are cancelled. + // TODO(ctiller): this should be remedied before promise based calls are + // dexperimentalized. + void AcceptTransportStatsFromContext() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + final_stats_ = *call_context_.call_stats(); + } + + grpc_completion_queue* cq() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { return cq_; } + + void CToMetadata(grpc_metadata* metadata, size_t count, + grpc_metadata_batch* batch); + + std::string ActivityDebugTag() const override { return DebugTag(); } + + // At the end of the call run any finalization actions. + void RunFinalization(grpc_status_code status, const char* status_details) { + grpc_call_final_info final_info; + final_info.stats = final_stats_; + final_info.final_status = status; + final_info.error_string = status_details; + finalization_.Run(&final_info); + } + + private: + union CompletionInfo { + struct Pending { + // Bitmask of PendingOps + uint8_t pending_op_bits; + bool is_closure; + bool success; + void* tag; + } pending; + grpc_cq_completion completion; + }; + + class NonOwningWakable final : public Wakeable { + public: + explicit NonOwningWakable(PromiseBasedCall* call) : call_(call) {} + + // Ref the Handle (not the activity). + void Ref() { refs_.fetch_add(1, std::memory_order_relaxed); } + + // Activity is going away... drop its reference and sever the connection + // back. + void DropActivity() ABSL_LOCKS_EXCLUDED(mu_) { + auto unref = absl::MakeCleanup([this]() { Unref(); }); + MutexLock lock(&mu_); + GPR_ASSERT(call_ != nullptr); + call_ = nullptr; + } + + // Activity needs to wake up (if it still exists!) - wake it up, and drop + // the ref that was kept for this handle. + void Wakeup() override ABSL_LOCKS_EXCLUDED(mu_) { + // Drop the ref to the handle at end of scope (we have one ref = one + // wakeup semantics). + auto unref = absl::MakeCleanup([this]() { Unref(); }); + ReleasableMutexLock lock(&mu_); + // Note that activity refcount can drop to zero, but we could win the lock + // against DropActivity, so we need to only increase activities refcount + // if it is non-zero. + if (call_ != nullptr && call_->RefIfNonZero()) { + PromiseBasedCall* call = call_; + lock.Release(); + // Activity still exists and we have a reference: wake it up, which will + // drop the ref. + call->Wakeup(); + } + } + + std::string ActivityDebugTag() const override { + MutexLock lock(&mu_); + return call_ == nullptr ? "" : call_->DebugTag(); + } + + void Drop() override { Unref(); } + + private: + // Unref the Handle (not the activity). + void Unref() { + if (1 == refs_.fetch_sub(1, std::memory_order_acq_rel)) { + delete this; + } + } + + mutable Mutex mu_; + // We have two initial refs: one for the wakeup that this is created for, + // and will be dropped by Wakeup, and the other for the activity which is + // dropped by DropActivity. + std::atomic refs_{2}; + PromiseBasedCall* call_ ABSL_GUARDED_BY(mu_); + }; + + static void OnDestroy(void* arg, grpc_error_handle) { + auto* call = static_cast(arg); + ScopedContext context(call); + call->DeleteThis(); + } + + // First 32 bits are strong refs, next 32 bits are weak refs. + static uint64_t MakeRefPair(uint32_t strong, uint32_t weak) { + return (static_cast(strong) << 32) + static_cast(weak); + } + static uint32_t GetStrongRefs(uint64_t ref_pair) { + return static_cast(ref_pair >> 32); + } + static uint32_t GetWeakRefs(uint64_t ref_pair) { + return static_cast(ref_pair & 0xffffffffu); + } + + bool RefIfNonZero() { + uint64_t prev_ref_pair = refs_.load(std::memory_order_acquire); + do { + const uint32_t strong_refs = GetStrongRefs(prev_ref_pair); + if (strong_refs == 0) return false; + } while (!refs_.compare_exchange_weak( + prev_ref_pair, prev_ref_pair + MakeRefPair(1, 0), + std::memory_order_acq_rel, std::memory_order_acquire)); + return true; + } + + mutable Mutex mu_; + std::atomic refs_{MakeRefPair(1, 0)}; + CallContext call_context_{this}; + bool keep_polling_ ABSL_GUARDED_BY(mu()) = false; + + /* Contexts for various subsystems (security, tracing, ...). */ + grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {}; + grpc_completion_queue* cq_ ABSL_GUARDED_BY(mu_); + FragmentAllocator fragment_allocator_ ABSL_GUARDED_BY(mu_); + NonOwningWakable* non_owning_wakeable_ ABSL_GUARDED_BY(mu_) = nullptr; + CompletionInfo completion_info_[6]; + grpc_call_stats final_stats_{}; + CallFinalization finalization_; +}; + +template +grpc_error_handle MakePromiseBasedCall(grpc_call_create_args* args, + grpc_call** out_call) { + Channel* channel = args->channel.get(); + + auto alloc = Arena::CreateWithAlloc(channel->CallSizeEstimate(), sizeof(T), + channel->allocator()); + PromiseBasedCall* call = new (alloc.second) T(alloc.first, args); + *out_call = call->c_ptr(); + GPR_DEBUG_ASSERT(Call::FromC(*out_call) == call); + return GRPC_ERROR_NONE; +} + +PromiseBasedCall::PromiseBasedCall(Arena* arena, + const grpc_call_create_args& args) + : Call(arena, args.server_transport_data == nullptr, args.send_deadline, + args.channel->Ref()), + cq_(args.cq) { + if (args.cq != nullptr) { + GPR_ASSERT(args.pollset_set_alternative == nullptr && + "Only one of 'cq' and 'pollset_set_alternative' should be " + "non-nullptr."); + GRPC_CQ_INTERNAL_REF(args.cq, "bind"); + call_context_.pollent_ = + grpc_polling_entity_create_from_pollset(grpc_cq_pollset(args.cq)); + } + if (args.pollset_set_alternative != nullptr) { + call_context_.pollent_ = grpc_polling_entity_create_from_pollset_set( + args.pollset_set_alternative); + } +} + +Waker PromiseBasedCall::MakeNonOwningWaker() { + if (non_owning_wakeable_ == nullptr) { + non_owning_wakeable_ = new NonOwningWakable(this); + } else { + non_owning_wakeable_->Ref(); + } + return Waker(non_owning_wakeable_); +} + +void PromiseBasedCall::CToMetadata(grpc_metadata* metadata, size_t count, + grpc_metadata_batch* b) { + for (size_t i = 0; i < count; i++) { + grpc_metadata* md = &metadata[i]; + auto key = StringViewFromSlice(md->key); + // Filter "content-length metadata" + if (key == "content-length") continue; + b->Append(key, Slice(CSliceRef(md->value)), + [md](absl::string_view error, const Slice& value) { + gpr_log(GPR_DEBUG, "Append error: %s", + absl::StrCat("key=", StringViewFromSlice(md->key), + " error=", error, + " value=", value.as_string_view()) + .c_str()); + }); + } +} + +void PromiseBasedCall::ContextSet(grpc_context_index elem, void* value, + void (*destroy)(void*)) { + if (context_[elem].destroy != nullptr) { + context_[elem].destroy(context_[elem].value); + } + context_[elem].value = value; + context_[elem].destroy = destroy; +} + +void* PromiseBasedCall::ContextGet(grpc_context_index elem) const { + return context_[elem].value; +} + +PromiseBasedCall::Completion PromiseBasedCall::StartCompletion( + void* tag, bool is_closure, const grpc_op* ops) { + Completion c(BatchSlotForOp(ops[0].op)); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sStartCompletion %s tag=%p", DebugTag().c_str(), + c.ToString().c_str(), tag); + } + if (!is_closure) { + grpc_cq_begin_op(cq(), tag); + } + completion_info_[c.index()].pending = { + PendingOpBit(PendingOp::kStartingBatch), is_closure, true, tag}; + return c; +} + +PromiseBasedCall::Completion PromiseBasedCall::AddOpToCompletion( + const Completion& completion, PendingOp reason) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sAddOpToCompletion %s %s", DebugTag().c_str(), + completion.ToString().c_str(), PendingOpString(reason)); + } + auto& pending_op_bits = + completion_info_[completion.index()].pending.pending_op_bits; + GPR_ASSERT((pending_op_bits & PendingOpBit(reason)) == 0); + pending_op_bits |= PendingOpBit(reason); + return Completion(completion.index()); +} + +void PromiseBasedCall::FailCompletion(const Completion& completion) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sFailCompletion %s", DebugTag().c_str(), + completion.ToString().c_str()); + } + completion_info_[completion.index()].pending.success = false; +} + +void PromiseBasedCall::FinishOpOnCompletion(Completion* completion, + PendingOp reason) { + if (grpc_call_trace.enabled()) { + auto pending_op_bits = + completion_info_[completion->index()].pending.pending_op_bits; + bool success = completion_info_[completion->index()].pending.success; + std::vector pending; + for (size_t i = 0; i < 8 * sizeof(pending_op_bits); i++) { + if (static_cast(i) == reason) continue; + if (pending_op_bits & (1 << i)) { + pending.push_back(PendingOpString(static_cast(i))); + } + } + gpr_log( + GPR_INFO, "%sFinishOpOnCompletion %s %s %s", DebugTag().c_str(), + completion->ToString().c_str(), PendingOpString(reason), + (pending.empty() + ? (success ? std::string("done") : std::string("failed")) + : absl::StrFormat("pending_ops={%s}", absl::StrJoin(pending, ","))) + .c_str()); + } + const uint8_t i = completion->TakeIndex(); + GPR_ASSERT(i < GPR_ARRAY_SIZE(completion_info_)); + CompletionInfo::Pending& pending = completion_info_[i].pending; + GPR_ASSERT(pending.pending_op_bits & PendingOpBit(reason)); + pending.pending_op_bits &= ~PendingOpBit(reason); + auto error = pending.success ? GRPC_ERROR_NONE : GRPC_ERROR_CANCELLED; + if (pending.pending_op_bits == 0) { + if (pending.is_closure) { + ExecCtx::Run(DEBUG_LOCATION, static_cast(pending.tag), + error); + } else { + grpc_cq_end_op( + cq(), pending.tag, error, [](void*, grpc_cq_completion*) {}, nullptr, + &completion_info_[i].completion); + } + } +} + +void PromiseBasedCall::Update() { + keep_polling_ = false; + do { + UpdateOnce(); + } while (std::exchange(keep_polling_, false)); +} + +void PromiseBasedCall::ForceImmediateRepoll() { keep_polling_ = true; } + +void PromiseBasedCall::SetCompletionQueue(grpc_completion_queue* cq) { + MutexLock lock(&mu_); + cq_ = cq; + GRPC_CQ_INTERNAL_REF(cq, "bind"); + call_context_.pollent_ = + grpc_polling_entity_create_from_pollset(grpc_cq_pollset(cq)); +} + +/////////////////////////////////////////////////////////////////////////////// +// CallContext + +void CallContext::RunInContext(absl::AnyInvocable fn) { + call_->RunInContext(std::move(fn)); +} + +void CallContext::IncrementRefCount(const char* reason) { + call_->InternalRef(reason); +} + +void CallContext::Unref(const char* reason) { call_->InternalUnref(reason); } + +/////////////////////////////////////////////////////////////////////////////// +// ClientPromiseBasedCall + +class ClientPromiseBasedCall final : public PromiseBasedCall { + public: + ClientPromiseBasedCall(Arena* arena, grpc_call_create_args* args) + : PromiseBasedCall(arena, *args) { + GRPC_STATS_INC_CLIENT_CALLS_CREATED(); + ScopedContext context(this); + send_initial_metadata_ = + GetContext()->MakeClientMetadata(); + send_initial_metadata_->Set(HttpPathMetadata(), std::move(*args->path)); + if (args->authority.has_value()) { + send_initial_metadata_->Set(HttpAuthorityMetadata(), + std::move(*args->authority)); + } + if (auto* channelz_channel = channel()->channelz_node()) { + channelz_channel->RecordCallStarted(); + } + } + + ~ClientPromiseBasedCall() override { + ScopedContext context(this); + send_initial_metadata_.reset(); + recv_status_on_client_ = absl::monostate(); + promise_ = ArenaPromise(); + // Need to destroy the pipes under the ScopedContext above, so we move them + // out here and then allow the destructors to run at end of scope, but + // before context. + auto c2s = std::move(client_to_server_messages_); + auto s2c = std::move(server_to_client_messages_); + } + + absl::string_view GetServerAuthority() const override { abort(); } + void CancelWithError(grpc_error_handle error) override; + bool Completed() override; + void Orphan() override { + MutexLock lock(mu()); + ScopedContext ctx(this); + if (!completed_) Finish(ServerMetadataHandle(absl::CancelledError())); + } + bool is_trailers_only() const override { + MutexLock lock(mu()); + return is_trailers_only_; + } + bool failed_before_recv_message() const override { abort(); } + + grpc_call_error StartBatch(const grpc_op* ops, size_t nops, void* notify_tag, + bool is_notify_tag_closure) override; + + std::string DebugTag() const override { + return absl::StrFormat("CLIENT_CALL[%p]: ", this); + } + + private: + // Poll the underlying promise (and sundry objects) once. + void UpdateOnce() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()) override; + // Finish the call with the given status/trailing metadata. + void Finish(ServerMetadataHandle trailing_metadata) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + // Validate that a set of ops is valid for a client call. + grpc_call_error ValidateBatch(const grpc_op* ops, size_t nops) const + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + // Commit a valid batch of operations to be executed. + void CommitBatch(const grpc_op* ops, size_t nops, + const Completion& completion) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + // Start the underlying promise. + void StartPromise(ClientMetadataHandle client_initial_metadata) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + // Publish some metadata out to the application. + static void PublishMetadataArray(grpc_metadata_array* array, + ServerMetadata* md); + // Publish status out to the application. + void PublishStatus( + grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args, + ServerMetadataHandle trailing_metadata) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + // Publish server initial metadata out to the application. + void PublishInitialMetadata(ServerMetadata* metadata) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu()); + + ArenaPromise promise_ ABSL_GUARDED_BY(mu()); + Latch server_initial_metadata_ ABSL_GUARDED_BY(mu()); + Pipe client_to_server_messages_ ABSL_GUARDED_BY(mu()){arena()}; + Pipe server_to_client_messages_ ABSL_GUARDED_BY(mu()){arena()}; + + ClientMetadataHandle send_initial_metadata_; + grpc_metadata_array* recv_initial_metadata_ ABSL_GUARDED_BY(mu()) = nullptr; + grpc_byte_buffer** recv_message_ ABSL_GUARDED_BY(mu()) = nullptr; + absl::variant + recv_status_on_client_ ABSL_GUARDED_BY(mu()); + absl::optional::PushType> outstanding_send_ + ABSL_GUARDED_BY(mu()); + absl::optional::NextType> outstanding_recv_ + ABSL_GUARDED_BY(mu()); + absl::optional::WaitPromise> + server_initial_metadata_ready_; + absl::optional incoming_compression_algorithm_; + Completion recv_initial_metadata_completion_ ABSL_GUARDED_BY(mu()); + Completion recv_status_on_client_completion_ ABSL_GUARDED_BY(mu()); + Completion send_message_completion_ ABSL_GUARDED_BY(mu()); + Completion recv_message_completion_ ABSL_GUARDED_BY(mu()); + bool completed_ ABSL_GUARDED_BY(mu()) = false; + bool is_trailers_only_ ABSL_GUARDED_BY(mu()); +}; + +void ClientPromiseBasedCall::StartPromise( + ClientMetadataHandle client_initial_metadata) { + GPR_ASSERT(!promise_.has_value()); + promise_ = channel()->channel_stack()->MakeClientCallPromise(CallArgs{ + std::move(client_initial_metadata), + &server_initial_metadata_, + &client_to_server_messages_.receiver, + &server_to_client_messages_.sender, + }); +} + +void ClientPromiseBasedCall::CancelWithError(grpc_error_handle error) { + MutexLock lock(mu()); + ScopedContext context(this); + Finish(ServerMetadataHandle(grpc_error_to_absl_status(error))); + GRPC_ERROR_UNREF(error); +} + +grpc_call_error ClientPromiseBasedCall::ValidateBatch(const grpc_op* ops, + size_t nops) const { + BitSet<8> got_ops; + for (size_t op_idx = 0; op_idx < nops; op_idx++) { + const grpc_op& op = ops[op_idx]; + switch (op.op) { + case GRPC_OP_SEND_INITIAL_METADATA: + if (!AreInitialMetadataFlagsValid(op.flags)) { + return GRPC_CALL_ERROR_INVALID_FLAGS; + } + if (!ValidateMetadata(op.data.send_initial_metadata.count, + op.data.send_initial_metadata.metadata)) { + return GRPC_CALL_ERROR_INVALID_METADATA; + } + break; + case GRPC_OP_SEND_MESSAGE: + if (!AreWriteFlagsValid(op.flags)) { + return GRPC_CALL_ERROR_INVALID_FLAGS; + } + break; + case GRPC_OP_RECV_INITIAL_METADATA: + case GRPC_OP_RECV_MESSAGE: + case GRPC_OP_SEND_CLOSE_FROM_CLIENT: + case GRPC_OP_RECV_STATUS_ON_CLIENT: + if (op.flags != 0) return GRPC_CALL_ERROR_INVALID_FLAGS; + break; + case GRPC_OP_RECV_CLOSE_ON_SERVER: + case GRPC_OP_SEND_STATUS_FROM_SERVER: + return GRPC_CALL_ERROR_NOT_ON_CLIENT; + } + if (got_ops.is_set(op.op)) return GRPC_CALL_ERROR_TOO_MANY_OPERATIONS; + got_ops.set(op.op); + } + return GRPC_CALL_OK; +} + +void ClientPromiseBasedCall::CommitBatch(const grpc_op* ops, size_t nops, + const Completion& completion) { + for (size_t op_idx = 0; op_idx < nops; op_idx++) { + const grpc_op& op = ops[op_idx]; + switch (op.op) { + case GRPC_OP_SEND_INITIAL_METADATA: { + // compression not implemented + GPR_ASSERT( + !op.data.send_initial_metadata.maybe_compression_level.is_set); + if (!completed_) { + CToMetadata(op.data.send_initial_metadata.metadata, + op.data.send_initial_metadata.count, + send_initial_metadata_.get()); + StartPromise(std::move(send_initial_metadata_)); + } + } break; + case GRPC_OP_RECV_INITIAL_METADATA: { + recv_initial_metadata_ = + op.data.recv_initial_metadata.recv_initial_metadata; + server_initial_metadata_ready_ = server_initial_metadata_.Wait(); + recv_initial_metadata_completion_ = + AddOpToCompletion(completion, PendingOp::kReceiveInitialMetadata); + } break; + case GRPC_OP_RECV_STATUS_ON_CLIENT: { + recv_status_on_client_completion_ = + AddOpToCompletion(completion, PendingOp::kReceiveStatusOnClient); + if (auto* finished_metadata = + absl::get_if(&recv_status_on_client_)) { + PublishStatus(op.data.recv_status_on_client, + std::move(*finished_metadata)); + } else { + recv_status_on_client_ = op.data.recv_status_on_client; + } + } break; + case GRPC_OP_SEND_MESSAGE: { + GPR_ASSERT(!outstanding_send_.has_value()); + if (!completed_) { + send_message_completion_ = + AddOpToCompletion(completion, PendingOp::kSendMessage); + SliceBuffer send; + grpc_slice_buffer_swap( + &op.data.send_message.send_message->data.raw.slice_buffer, + send.c_slice_buffer()); + outstanding_send_.emplace(client_to_server_messages_.sender.Push( + GetContext()->MakeMessage(std::move(send), + op.flags))); + } else { + FailCompletion(completion); + } + } break; + case GRPC_OP_RECV_MESSAGE: { + GPR_ASSERT(!outstanding_recv_.has_value()); + recv_message_ = op.data.recv_message.recv_message; + recv_message_completion_ = + AddOpToCompletion(completion, PendingOp::kReceiveMessage); + outstanding_recv_.emplace(server_to_client_messages_.receiver.Next()); + } break; + case GRPC_OP_SEND_CLOSE_FROM_CLIENT: { + client_to_server_messages_.sender.Close(); + } break; + case GRPC_OP_SEND_STATUS_FROM_SERVER: + case GRPC_OP_RECV_CLOSE_ON_SERVER: + abort(); // unreachable + } + } +} + +grpc_call_error ClientPromiseBasedCall::StartBatch(const grpc_op* ops, + size_t nops, + void* notify_tag, + bool is_notify_tag_closure) { + MutexLock lock(mu()); + ScopedContext activity_context(this); + if (nops == 0) { + EndOpImmediately(cq(), notify_tag, is_notify_tag_closure); + return GRPC_CALL_OK; + } + const grpc_call_error validation_result = ValidateBatch(ops, nops); + if (validation_result != GRPC_CALL_OK) { + return validation_result; + } + Completion completion = + StartCompletion(notify_tag, is_notify_tag_closure, ops); + CommitBatch(ops, nops, completion); + Update(); + FinishOpOnCompletion(&completion, PendingOp::kStartingBatch); + return GRPC_CALL_OK; +} + +void ClientPromiseBasedCall::PublishInitialMetadata(ServerMetadata* metadata) { + incoming_compression_algorithm_ = + metadata->Take(GrpcEncodingMetadata()).value_or(GRPC_COMPRESS_NONE); + server_initial_metadata_ready_.reset(); + GPR_ASSERT(recv_initial_metadata_ != nullptr); + PublishMetadataArray(std::exchange(recv_initial_metadata_, nullptr), + metadata); + FinishOpOnCompletion(&recv_initial_metadata_completion_, + PendingOp::kReceiveInitialMetadata); +} + +void ClientPromiseBasedCall::UpdateOnce() { + if (grpc_call_trace.enabled()) { + auto present_and_completion_text = + [](const char* caption, bool has, + const Completion& completion) -> std::string { + if (has) { + if (completion.has_value()) { + return absl::StrCat(caption, ":", + static_cast(completion.index()), " "); + } else { + return absl::StrCat(caption, + ":!!BUG:operation is present, no completion!! "); + } + } else { + if (!completion.has_value()) { + return ""; + } else { + return absl::StrCat( + caption, ":no-op:", static_cast(completion.index()), " "); + } + } + }; + gpr_log( + GPR_INFO, "%sUpdateOnce: %s%s%shas_promise=%s", DebugTag().c_str(), + present_and_completion_text("server_initial_metadata_ready", + server_initial_metadata_ready_.has_value(), + recv_initial_metadata_completion_) + .c_str(), + present_and_completion_text("outstanding_send", + outstanding_send_.has_value(), + send_message_completion_) + .c_str(), + present_and_completion_text("outstanding_recv", + outstanding_recv_.has_value(), + recv_message_completion_) + .c_str(), + promise_.has_value() ? "true" : "false"); + } + if (send_message_completion_.has_value()) { + FinishOpOnCompletion(&send_message_completion_, PendingOp::kSendMessage); + } + if (server_initial_metadata_ready_.has_value()) { + Poll r = (*server_initial_metadata_ready_)(); + if (ServerMetadata*** server_initial_metadata = + absl::get_if(&r)) { + PublishInitialMetadata(**server_initial_metadata); + } else if (completed_) { + ServerMetadata no_metadata{GetContext()}; + PublishInitialMetadata(&no_metadata); + } + } + if (outstanding_send_.has_value()) { + Poll r = (*outstanding_send_)(); + if (const bool* result = absl::get_if(&r)) { + outstanding_send_.reset(); + if (!*result) { + FailCompletion(send_message_completion_); + Finish(ServerMetadataHandle(absl::Status( + absl::StatusCode::kInternal, "Failed to send message to server"))); + } + } + } + if (promise_.has_value()) { + Poll r = promise_(); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sUpdateOnce: promise returns %s", DebugTag().c_str(), + PollToString(r, [](const ServerMetadataHandle& h) { + return h->DebugString(); + }).c_str()); + } + if (auto* result = absl::get_if(&r)) { + AcceptTransportStatsFromContext(); + Finish(std::move(*result)); + } + } + if (incoming_compression_algorithm_.has_value() && + outstanding_recv_.has_value()) { + Poll> r = (*outstanding_recv_)(); + if (auto* result = absl::get_if>(&r)) { + outstanding_recv_.reset(); + if (result->has_value()) { + MessageHandle& message = **result; + if ((message->flags() & GRPC_WRITE_INTERNAL_COMPRESS) && + (incoming_compression_algorithm_ != GRPC_COMPRESS_NONE)) { + *recv_message_ = grpc_raw_compressed_byte_buffer_create( + nullptr, 0, *incoming_compression_algorithm_); + } else { + *recv_message_ = grpc_raw_byte_buffer_create(nullptr, 0); + } + grpc_slice_buffer_move_into(message->payload()->c_slice_buffer(), + &(*recv_message_)->data.raw.slice_buffer); + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sUpdateOnce: outstanding_recv finishes: received %" PRIdPTR + " byte message", + DebugTag().c_str(), + (*recv_message_)->data.raw.slice_buffer.length); + } + } else { + if (grpc_call_trace.enabled()) { + gpr_log( + GPR_INFO, + "%sUpdateOnce: outstanding_recv finishes: received end-of-stream", + DebugTag().c_str()); + } + *recv_message_ = nullptr; + } + FinishOpOnCompletion(&recv_message_completion_, + PendingOp::kReceiveMessage); + } else if (completed_) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, + "%sUpdateOnce: outstanding_recv finishes: promise has " + "completed without queuing a message, forcing end-of-stream", + DebugTag().c_str()); + } + outstanding_recv_.reset(); + *recv_message_ = nullptr; + FinishOpOnCompletion(&recv_message_completion_, + PendingOp::kReceiveMessage); + } + } +} + +void ClientPromiseBasedCall::Finish(ServerMetadataHandle trailing_metadata) { + if (grpc_call_trace.enabled()) { + gpr_log(GPR_INFO, "%sFinish: %s", DebugTag().c_str(), + trailing_metadata->DebugString().c_str()); + } + promise_ = ArenaPromise(); + completed_ = true; + if (recv_initial_metadata_ != nullptr) { + ForceImmediateRepoll(); + } + const bool pending_initial_metadata = + server_initial_metadata_ready_.has_value(); + server_initial_metadata_ready_.reset(); + Poll r = server_initial_metadata_.Wait()(); + if (auto* result = absl::get_if(&r)) { + if (pending_initial_metadata) PublishInitialMetadata(**result); + is_trailers_only_ = false; + } else { + if (pending_initial_metadata) { + ServerMetadata no_metadata{GetContext()}; + PublishInitialMetadata(&no_metadata); + } + is_trailers_only_ = true; + } + if (auto* channelz_channel = channel()->channelz_node()) { + if (trailing_metadata->get(GrpcStatusMetadata()) + .value_or(GRPC_STATUS_UNKNOWN) == GRPC_STATUS_OK) { + channelz_channel->RecordCallSucceeded(); + } else { + channelz_channel->RecordCallFailed(); + } + } + if (auto* status_request = + absl::get_if( + &recv_status_on_client_)) { + PublishStatus(*status_request, std::move(trailing_metadata)); + } else { + recv_status_on_client_ = std::move(trailing_metadata); + } +} + +namespace { +std::string MakeErrorString(const ServerMetadata* trailing_metadata) { + std::string out = absl::StrCat( + trailing_metadata->get(GrpcStatusFromWire()).value_or(false) + ? "Error received from peer" + : "Error generated by client", + "grpc_status: ", + grpc_status_code_to_string(trailing_metadata->get(GrpcStatusMetadata()) + .value_or(GRPC_STATUS_UNKNOWN))); + if (const Slice* message = + trailing_metadata->get_pointer(GrpcMessageMetadata())) { + absl::StrAppend(&out, "\ngrpc_message: ", message->as_string_view()); + } + if (auto annotations = trailing_metadata->get_pointer(GrpcStatusContext())) { + absl::StrAppend(&out, "\nStatus Context:"); + for (const std::string& annotation : *annotations) { + absl::StrAppend(&out, "\n ", annotation); + } + } + return out; +} +} // namespace + +void ClientPromiseBasedCall::PublishStatus( + grpc_op::grpc_op_data::grpc_op_recv_status_on_client op_args, + ServerMetadataHandle trailing_metadata) { + const grpc_status_code status = trailing_metadata->get(GrpcStatusMetadata()) + .value_or(GRPC_STATUS_UNKNOWN); + *op_args.status = status; + absl::string_view message_string; + if (Slice* message = trailing_metadata->get_pointer(GrpcMessageMetadata())) { + message_string = message->as_string_view(); + *op_args.status_details = message->Ref().TakeCSlice(); + } else { + *op_args.status_details = grpc_empty_slice(); + } + if (message_string.empty()) { + RunFinalization(status, nullptr); + } else { + std::string error_string(message_string); + RunFinalization(status, error_string.c_str()); + } + if (op_args.error_string != nullptr && status != GRPC_STATUS_OK) { + *op_args.error_string = + gpr_strdup(MakeErrorString(trailing_metadata.get()).c_str()); + } + PublishMetadataArray(op_args.trailing_metadata, trailing_metadata.get()); + FinishOpOnCompletion(&recv_status_on_client_completion_, + PendingOp::kReceiveStatusOnClient); +} + +void ClientPromiseBasedCall::PublishMetadataArray(grpc_metadata_array* array, + ServerMetadata* md) { + const auto md_count = md->count(); + if (md_count > array->capacity) { + array->capacity = + std::max(array->capacity + md->count(), array->capacity * 3 / 2); + array->metadata = static_cast( + gpr_realloc(array->metadata, sizeof(grpc_metadata) * array->capacity)); + } + PublishToAppEncoder encoder(array); + md->Encode(&encoder); +} + +bool ClientPromiseBasedCall::Completed() { + MutexLock lock(mu()); + return completed_; +} + +gpr_atm* CallContext::peer_string_atm_ptr() { + return call_->peer_string_atm_ptr(); +} + } // namespace grpc_core +/////////////////////////////////////////////////////////////////////////////// +// C-based API + void* grpc_call_arena_alloc(grpc_call* call, size_t size) { grpc_core::ExecCtx exec_ctx; return grpc_core::Call::FromC(call)->arena()->Alloc(size); @@ -1742,6 +2879,13 @@ size_t grpc_call_get_initial_size_estimate() { grpc_error_handle grpc_call_create(grpc_call_create_args* args, grpc_call** out_call) { + if (grpc_core::IsPromiseBasedClientCallEnabled() && + args->channel->is_promising()) { + if (args->server_transport_data == nullptr) { + return grpc_core::MakePromiseBasedCall( + args, out_call); + } + } return grpc_core::FilterStackCall::Create(args, out_call); } @@ -1753,6 +2897,7 @@ void grpc_call_set_completion_queue(grpc_call* call, void grpc_call_ref(grpc_call* c) { grpc_core::Call::FromC(c)->ExternalRef(); } void grpc_call_unref(grpc_call* c) { + grpc_core::ExecCtx exec_ctx; grpc_core::Call::FromC(c)->ExternalUnref(); } diff --git a/src/core/lib/surface/call.h b/src/core/lib/surface/call.h index 944808867b8..5e13178f46b 100644 --- a/src/core/lib/surface/call.h +++ b/src/core/lib/surface/call.h @@ -24,14 +24,17 @@ #include #include +#include "absl/functional/any_invocable.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include #include +#include #include #include "src/core/lib/channel/channel_fwd.h" +#include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/context.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" @@ -39,6 +42,8 @@ #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/iomgr_fwd.h" +#include "src/core/lib/iomgr/polling_entity.h" +#include "src/core/lib/promise/context.h" #include "src/core/lib/resource_quota/arena.h" #include "src/core/lib/slice/slice.h" #include "src/core/lib/surface/api_trace.h" @@ -67,6 +72,45 @@ typedef struct grpc_call_create_args { grpc_core::Timestamp send_deadline; } grpc_call_create_args; +namespace grpc_core { +class PromiseBasedCall; + +// TODO(ctiller): move more call things into this type +class CallContext { + public: + explicit CallContext(PromiseBasedCall* call) : call_(call) {} + + // Run some action in the call activity context. This is needed to adapt some + // legacy systems to promises, and will likely disappear once that conversion + // is complete. + void RunInContext(absl::AnyInvocable fn); + + // TODO(ctiller): remove this once transport APIs are promise based + void IncrementRefCount(const char* reason = "call_context"); + + // TODO(ctiller): remove this once transport APIs are promise based + void Unref(const char* reason = "call_context"); + + grpc_call_stats* call_stats() { return &call_stats_; } + gpr_atm* peer_string_atm_ptr(); + grpc_polling_entity* polling_entity() { return &pollent_; } + + private: + friend class PromiseBasedCall; + // Call final info. + grpc_call_stats call_stats_; + // Pollset stuff, can't wait to remove. + // TODO(ctiller): bring forth EventEngine. + grpc_polling_entity pollent_; + // TODO(ctiller): remove this once transport APIs are promise based and we + // don't need refcounting here. + PromiseBasedCall* const call_; +}; + +template <> +struct ContextType {}; +} // namespace grpc_core + /* Create a new call based on \a args. Regardless of success or failure, always returns a valid new call into *call */ diff --git a/src/core/lib/surface/call_trace.cc b/src/core/lib/surface/call_trace.cc new file mode 100644 index 00000000000..e234c2d7004 --- /dev/null +++ b/src/core/lib/surface/call_trace.cc @@ -0,0 +1,114 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "src/core/lib/surface/call_trace.h" + +#include +#include +#include +#include + +#include "absl/base/thread_annotations.h" +#include "absl/container/flat_hash_map.h" +#include "absl/meta/type_traits.h" +#include "absl/status/status.h" +#include "absl/types/variant.h" + +#include + +#include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/gprpp/no_destruct.h" +#include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/promise/activity.h" +#include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/transport/call_fragments.h" +#include "src/core/lib/transport/metadata_batch.h" +#include "src/core/lib/transport/transport.h" + +namespace grpc_core { + +const grpc_channel_filter* PromiseTracingFilterFor( + const grpc_channel_filter* filter) { + struct DerivedFilter : public grpc_channel_filter { + explicit DerivedFilter(const grpc_channel_filter* filter) + : grpc_channel_filter{ + /* start_transport_stream_op_batch: */ grpc_call_next_op, + /* make_call_promise: */ + [](grpc_channel_element* elem, CallArgs call_args, + NextPromiseFactory next_promise_factory) + -> ArenaPromise { + auto* source_filter = + static_cast(elem->filter)->filter; + gpr_log( + GPR_DEBUG, + "%sCreateCallPromise[%s]: client_initial_metadata=%s", + Activity::current()->DebugTag().c_str(), + source_filter->name, + call_args.client_initial_metadata->DebugString().c_str()); + return [source_filter, child = next_promise_factory( + std::move(call_args))]() mutable { + gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: begin", + Activity::current()->DebugTag().c_str(), + source_filter->name); + auto r = child(); + if (auto* p = absl::get_if(&r)) { + gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: done: %s", + Activity::current()->DebugTag().c_str(), + source_filter->name, (*p)->DebugString().c_str()); + } else { + gpr_log(GPR_DEBUG, "%sPollCallPromise[%s]: <", + Activity::current()->DebugTag().c_str(), + source_filter->name); + } + return r; + }; + }, + grpc_channel_next_op, /* sizeof_call_data: */ 0, + /* init_call_elem: */ + [](grpc_call_element*, const grpc_call_element_args*) { + return absl::OkStatus(); + }, + grpc_call_stack_ignore_set_pollset_or_pollset_set, + /* destroy_call_elem: */ + [](grpc_call_element*, const grpc_call_final_info*, + grpc_closure*) {}, + /* sizeof_channel_data: */ 0, /* init_channel_elem: */ + [](grpc_channel_element*, grpc_channel_element_args*) { + return absl::OkStatus(); + }, + /* post_init_channel_elem: */ + [](grpc_channel_stack*, grpc_channel_element*) {}, + /* destroy_channel_elem: */ [](grpc_channel_element*) {}, + grpc_channel_next_get_info, filter->name}, + filter(filter) {} + const grpc_channel_filter* const filter; + }; + struct Globals { + Mutex mu; + absl::flat_hash_map> + map ABSL_GUARDED_BY(mu); + }; + auto* globals = NoDestructSingleton::Get(); + MutexLock lock(&globals->mu); + auto it = globals->map.find(filter); + if (it != globals->map.end()) return it->second.get(); + return globals->map.emplace(filter, std::make_unique(filter)) + .first->second.get(); +} + +} // namespace grpc_core diff --git a/src/core/lib/surface/call_trace.h b/src/core/lib/surface/call_trace.h new file mode 100644 index 00000000000..411aaf82400 --- /dev/null +++ b/src/core/lib/surface/call_trace.h @@ -0,0 +1,30 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_CORE_LIB_SURFACE_CALL_TRACE_H +#define GRPC_CORE_LIB_SURFACE_CALL_TRACE_H + +#include + +#include "src/core/lib/channel/channel_fwd.h" +#include "src/core/lib/debug/trace.h" + +extern grpc_core::TraceFlag grpc_call_trace; + +namespace grpc_core { +const grpc_channel_filter* PromiseTracingFilterFor( + const grpc_channel_filter* filter); +} + +#endif // GRPC_CORE_LIB_SURFACE_CALL_TRACE_H diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index aa24924e1a2..2d13f2cfcf3 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -62,11 +62,12 @@ namespace grpc_core { -Channel::Channel(bool is_client, std::string target, +Channel::Channel(bool is_client, bool is_promising, std::string target, const ChannelArgs& channel_args, grpc_compression_options compression_options, RefCountedPtr channel_stack) : is_client_(is_client), + is_promising_(is_promising), compression_options_(compression_options), call_size_estimate_(channel_stack->call_stack_size + grpc_call_get_initial_size_estimate()), @@ -150,8 +151,8 @@ absl::StatusOr> Channel::CreateWithBuilder( return RefCountedPtr(new Channel( grpc_channel_stack_type_is_client(builder->channel_stack_type()), - std::string(builder->target()), channel_args, compression_options, - std::move(*r))); + builder->IsPromising(), std::string(builder->target()), channel_args, + compression_options, std::move(*r))); } namespace { diff --git a/src/core/lib/surface/channel.h b/src/core/lib/surface/channel.h index 8914a200980..f1c5bb0a352 100644 --- a/src/core/lib/surface/channel.h +++ b/src/core/lib/surface/channel.h @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -34,6 +35,7 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include #include #include #include @@ -44,6 +46,7 @@ #include "src/core/lib/channel/channel_stack.h" // IWYU pragma: keep #include "src/core/lib/channel/channel_stack_builder.h" #include "src/core/lib/channel/channelz.h" +#include "src/core/lib/event_engine/default_event_engine.h" #include "src/core/lib/gprpp/cpp_impl_of.h" #include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/ref_counted.h" @@ -141,6 +144,7 @@ class Channel : public RefCounted, absl::string_view target() const { return target_; } MemoryAllocator* allocator() { return &allocator_; } bool is_client() const { return is_client_; } + bool is_promising() const { return is_promising_; } RegisteredCall* RegisterCall(const char* method, const char* host); int TestOnlyRegisteredCalls() { @@ -153,12 +157,18 @@ class Channel : public RefCounted, return registration_table_.method_registration_attempts; } + grpc_event_engine::experimental::EventEngine* event_engine() const { + return event_engine_.get(); + } + private: - Channel(bool is_client, std::string target, const ChannelArgs& channel_args, + Channel(bool is_client, bool is_promising, std::string target, + const ChannelArgs& channel_args, grpc_compression_options compression_options, RefCountedPtr channel_stack); const bool is_client_; + const bool is_promising_; const grpc_compression_options compression_options_; std::atomic call_size_estimate_; CallRegistrationTable registration_table_; @@ -166,6 +176,8 @@ class Channel : public RefCounted, MemoryAllocator allocator_; std::string target_; const RefCountedPtr channel_stack_; + const std::shared_ptr + event_engine_ = grpc_event_engine::experimental::GetDefaultEventEngine(); }; } // namespace grpc_core diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc index d6af3254c22..6628d63089d 100644 --- a/src/core/lib/surface/lame_client.cc +++ b/src/core/lib/surface/lame_client.cc @@ -41,10 +41,12 @@ #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/promise/pipe.h" #include "src/core/lib/promise/promise.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/transport.h" @@ -72,7 +74,10 @@ LameClientFilter::State::State() : state_tracker("lame_client", GRPC_CHANNEL_SHUTDOWN) {} ArenaPromise LameClientFilter::MakeCallPromise( - CallArgs, NextPromiseFactory) { + CallArgs args, NextPromiseFactory) { + // TODO(ctiller): remove if check once promise_based_filter is removed (Close + // is still needed) + if (args.incoming_messages != nullptr) args.incoming_messages->Close(); return Immediate(ServerMetadataHandle(error_)); } diff --git a/src/core/lib/surface/lame_client.h b/src/core/lib/surface/lame_client.h index ef5a8d6247a..8f048866f5f 100644 --- a/src/core/lib/surface/lame_client.h +++ b/src/core/lib/surface/lame_client.h @@ -35,6 +35,7 @@ #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/promise/arena_promise.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/transport.h" diff --git a/src/core/lib/transport/call_fragments.cc b/src/core/lib/transport/call_fragments.cc new file mode 100644 index 00000000000..1fb6941294d --- /dev/null +++ b/src/core/lib/transport/call_fragments.cc @@ -0,0 +1,45 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "src/core/lib/transport/call_fragments.h" + +namespace grpc_core { + +FragmentAllocator::Node* FragmentAllocator::AllocateNode() { + if (free_list_ != nullptr) { + Node* node = free_list_; + free_list_ = free_list_->next_free; + return node; + } + return static_cast(GetContext()->Alloc(sizeof(Node))); +} + +void FragmentAllocator::FreeNode(Node* node) { + node->next_free = free_list_; + free_list_ = node; +} + +void FragmentAllocator::Delete(grpc_metadata_batch* p) { + p->~grpc_metadata_batch(); + FreeNode(reinterpret_cast(p)); +} + +void FragmentAllocator::Delete(Message* m) { + m->~Message(); + FreeNode(reinterpret_cast(m)); +} + +} // namespace grpc_core diff --git a/src/core/lib/transport/call_fragments.h b/src/core/lib/transport/call_fragments.h new file mode 100644 index 00000000000..3c44b55c918 --- /dev/null +++ b/src/core/lib/transport/call_fragments.h @@ -0,0 +1,232 @@ +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GRPC_CORE_LIB_TRANSPORT_CALL_FRAGMENTS_H +#define GRPC_CORE_LIB_TRANSPORT_CALL_FRAGMENTS_H + +#include + +#include + +#include +#include + +#include "absl/status/status.h" +#include "absl/types/optional.h" + +#include + +#include "src/core/lib/promise/context.h" +#include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/slice/slice.h" +#include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/metadata_batch.h" + +namespace grpc_core { + +// TODO(ctiller): eliminate once MetadataHandle is constructable directly. +namespace promise_filter_detail { +class BaseCallData; +} // namespace promise_filter_detail + +class FragmentAllocator; + +// Small owned "handle" type to ensure one accessor at a time to metadata. +// The focus here is to get promises to use the syntax we'd like - we'll +// probably substitute some other smart pointer later. +template +class FragmentHandle { + public: + FragmentHandle() = default; + + FragmentHandle(const FragmentHandle&) = delete; + FragmentHandle& operator=(const FragmentHandle&) = delete; + + FragmentHandle(FragmentHandle&& other) noexcept + : handle_(other.handle_), + allocated_by_allocator_(other.allocated_by_allocator_) { + other.handle_ = nullptr; + other.allocated_by_allocator_ = false; + } + FragmentHandle& operator=(FragmentHandle&& other) noexcept { + DestroyHandle(); + handle_ = other.handle_; + allocated_by_allocator_ = other.allocated_by_allocator_; + other.handle_ = nullptr; + other.allocated_by_allocator_ = false; + return *this; + } + + explicit FragmentHandle(const absl::Status& status); + + ~FragmentHandle() { DestroyHandle(); } + + T* operator->() const { return handle_; } + bool has_value() const { return handle_ != nullptr; } + T* get() const { return handle_; } + void reset() { *this = FragmentHandle(); } + + static FragmentHandle TestOnlyWrap(T* p) { return FragmentHandle(p, false); } + + private: + // We restrict access to construction from a pointer to limit the number of + // cases that need dealing with as this code evolves. + friend class promise_filter_detail::BaseCallData; + friend class FragmentAllocator; + + explicit FragmentHandle(T* handle, bool allocated_by_allocator) + : handle_(handle), allocated_by_allocator_(allocated_by_allocator) {} + + void DestroyHandle(); + + T* Unwrap() { + T* result = handle_; + handle_ = nullptr; + return result; + } + + T* handle_ = nullptr; + // TODO(ctiller): remove this once promise_based_filter goes away. + // This bit determines whether the pointer is allocated by a metadata + // allocator or some other system. If it's held by a metadata allocator, we'll + // release it back when we're done with it. + bool allocated_by_allocator_ = false; +}; + +// Server metadata type +// TODO(ctiller): This should be a bespoke instance of MetadataMap<> +using ServerMetadata = grpc_metadata_batch; +using ServerMetadataHandle = FragmentHandle; + +// Client initial metadata type +// TODO(ctiller): This should be a bespoke instance of MetadataMap<> +using ClientMetadata = grpc_metadata_batch; +using ClientMetadataHandle = FragmentHandle; + +class Message { + public: + Message() = default; + ~Message() = default; + Message(SliceBuffer payload, uint32_t flags) + : payload_(std::move(payload)), flags_(flags) {} + Message(const Message&) = delete; + Message& operator=(const Message&) = delete; + + uint32_t flags() const { return flags_; } + SliceBuffer* payload() { return &payload_; } + const SliceBuffer* payload() const { return &payload_; } + + private: + SliceBuffer payload_; + uint32_t flags_ = 0; +}; + +using MessageHandle = FragmentHandle; + +// Ok/not-ok check for trailing metadata, so that it can be used as result types +// for TrySeq. +inline bool IsStatusOk(const ServerMetadataHandle& m) { + return m->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN) == + GRPC_STATUS_OK; +} + +// Within a call arena we need metadata at least four times - (client,server) x +// (initial,trailing), and possibly more for early returning promises. +// Since we often don't need these *simultaneously*, we can save memory by +// allocating/releasing them. +// We'd still like the memory to be part of the arena though, so this type +// creates a small free list of metadata objects and a central (call context) +// based place to create/destroy them. +class FragmentAllocator { + public: + FragmentAllocator() = default; + ~FragmentAllocator() = default; + FragmentAllocator(const FragmentAllocator&) = delete; + FragmentAllocator& operator=(const FragmentAllocator&) = delete; + + ClientMetadataHandle MakeClientMetadata() { + auto* node = AllocateNode(); + // TODO(ctiller): once we finish the promise transition, have metadata map + // know about arena contexts and allocate directly from there. + // (we could do so before, but there's enough places where we don't have a + // promise context up that it's too much whackamole) + new (&node->batch) ClientMetadata(GetContext()); + return ClientMetadataHandle(&node->batch, true); + } + + ServerMetadataHandle MakeServerMetadata() { return MakeClientMetadata(); } + + template + MessageHandle MakeMessage(Args&&... args) { + auto* node = AllocateNode(); + new (&node->message) Message(std::forward(args)...); + return MessageHandle(&node->message, true); + } + + private: + union Node { + Node* next_free; + grpc_metadata_batch batch; + Message message; + }; + + template + friend class FragmentHandle; + + void Delete(grpc_metadata_batch* p); + void Delete(Message* m); + + Node* AllocateNode(); + void FreeNode(Node* node); + + Node* free_list_ = nullptr; +}; + +template <> +struct ContextType {}; + +template +FragmentHandle::FragmentHandle(const absl::Status& status) { + // TODO(ctiller): currently we guarantee that MetadataAllocator is only + // present for promise based calls, and if we're using promise_based_filter + // it's not present. If we're in a promise based call, the correct thing is to + // use the metadata allocator to track the memory we need. If we're not, we + // need to do the hacky thing promise_based_filter does. + // This all goes away when promise_based_filter goes away, and this code will + // just assume there's an allocator present and move forward. + if (auto* allocator = GetContext()) { + handle_ = nullptr; + allocated_by_allocator_ = false; + *this = allocator->MakeServerMetadata(); + } else { + handle_ = GetContext()->New(GetContext()); + allocated_by_allocator_ = false; + } + handle_->Set(GrpcStatusMetadata(), + static_cast(status.code())); + if (status.ok()) return; + handle_->Set(GrpcMessageMetadata(), + Slice::FromCopiedString(status.message())); +} + +template +void FragmentHandle::DestroyHandle() { + if (allocated_by_allocator_) { + GetContext()->Delete(handle_); + } +} + +} // namespace grpc_core + +#endif // GRPC_CORE_LIB_TRANSPORT_CALL_FRAGMENTS_H diff --git a/src/core/lib/transport/metadata_batch.h b/src/core/lib/transport/metadata_batch.h index b8017378048..4308e3f716b 100644 --- a/src/core/lib/transport/metadata_batch.h +++ b/src/core/lib/transport/metadata_batch.h @@ -387,6 +387,14 @@ struct GrpcStatusContext { static const std::string& DisplayValue(const std::string& x); }; +// Annotation added by a transport to note that the status came from the wire. +struct GrpcStatusFromWire { + static absl::string_view DebugKey() { return "GrpcStatusFromWire"; } + static constexpr bool kRepeatable = false; + using ValueType = bool; + static absl::string_view DisplayValue(bool x) { return x ? "true" : "false"; } +}; + // Annotation added by client surface code to denote wait-for-ready state struct WaitForReady { struct ValueType { @@ -645,6 +653,13 @@ struct AdaptDisplayValueToLog { static std::string ToString(const std::string& value) { return value; } }; +template <> +struct AdaptDisplayValueToLog { + static std::string ToString(absl::string_view value) { + return std::string(value); + } +}; + template <> struct AdaptDisplayValueToLog { static std::string ToString(Slice value) { @@ -1301,7 +1316,8 @@ using grpc_metadata_batch_base = grpc_core::MetadataMap< grpc_core::LbCostBinMetadata, grpc_core::LbTokenMetadata, // Non-encodable things grpc_core::GrpcStreamNetworkState, grpc_core::PeerString, - grpc_core::GrpcStatusContext, grpc_core::WaitForReady>; + grpc_core::GrpcStatusContext, grpc_core::GrpcStatusFromWire, + grpc_core::WaitForReady>; struct grpc_metadata_batch : public grpc_metadata_batch_base { using grpc_metadata_batch_base::grpc_metadata_batch_base; diff --git a/src/core/lib/transport/transport.cc b/src/core/lib/transport/transport.cc index 23f75fa380d..e2670095168 100644 --- a/src/core/lib/transport/transport.cc +++ b/src/core/lib/transport/transport.cc @@ -24,6 +24,8 @@ #include +#include "absl/status/status.h" + #include "src/core/lib/gpr/alloc.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/executor.h" @@ -183,6 +185,36 @@ void grpc_transport_stream_op_batch_queue_finish_with_failure( } } +void grpc_transport_stream_op_batch_finish_with_failure_from_transport( + grpc_transport_stream_op_batch* batch, grpc_error_handle error) { + if (batch->cancel_stream) { + GRPC_ERROR_UNREF(batch->payload->cancel_stream.cancel_error); + } + // Construct a list of closures to execute. + if (batch->recv_initial_metadata) { + grpc_core::ExecCtx::Run( + DEBUG_LOCATION, + batch->payload->recv_initial_metadata.recv_initial_metadata_ready, + GRPC_ERROR_REF(error)); + } + if (batch->recv_message) { + grpc_core::ExecCtx::Run(DEBUG_LOCATION, + batch->payload->recv_message.recv_message_ready, + GRPC_ERROR_REF(error)); + } + if (batch->recv_trailing_metadata) { + grpc_core::ExecCtx::Run( + DEBUG_LOCATION, + batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready, + GRPC_ERROR_REF(error)); + } + if (batch->on_complete != nullptr) { + grpc_core::ExecCtx::Run(DEBUG_LOCATION, batch->on_complete, + GRPC_ERROR_REF(error)); + } + GRPC_ERROR_UNREF(error); +} + struct made_transport_op { grpc_closure outer_on_complete; grpc_closure* inner_on_complete = nullptr; diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index aef77dfc550..22c3ec2220f 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -28,7 +28,6 @@ #include #include -#include "absl/status/status.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -50,11 +49,11 @@ #include "src/core/lib/iomgr/iomgr_fwd.h" #include "src/core/lib/iomgr/polling_entity.h" #include "src/core/lib/promise/arena_promise.h" -#include "src/core/lib/promise/context.h" #include "src/core/lib/promise/latch.h" +#include "src/core/lib/promise/pipe.h" #include "src/core/lib/resource_quota/arena.h" -#include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice_buffer.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/connectivity_state.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport_fwd.h" @@ -81,83 +80,21 @@ struct grpc_transport_stream_op_batch_payload; (GRPC_WRITE_INTERNAL_COMPRESS | GRPC_WRITE_INTERNAL_TEST_ONLY_WAS_COMPRESSED) namespace grpc_core { -// TODO(ctiller): eliminate once MetadataHandle is constructable directly. -namespace promise_filter_detail { -class BaseCallData; -} - -// Small unowned "handle" type to ensure one accessor at a time to metadata. -// The focus here is to get promises to use the syntax we'd like - we'll -// probably substitute some other smart pointer later. -template -class MetadataHandle { - public: - MetadataHandle() = default; - - MetadataHandle(const MetadataHandle&) = delete; - MetadataHandle& operator=(const MetadataHandle&) = delete; - - MetadataHandle(MetadataHandle&& other) noexcept : handle_(other.handle_) { - other.handle_ = nullptr; - } - MetadataHandle& operator=(MetadataHandle&& other) noexcept { - handle_ = other.handle_; - other.handle_ = nullptr; - return *this; - } - - explicit MetadataHandle(const absl::Status& status) { - handle_ = GetContext()->New(GetContext()); - handle_->Set(GrpcStatusMetadata(), - static_cast(status.code())); - if (status.ok()) return; - handle_->Set(GrpcMessageMetadata(), - Slice::FromCopiedString(status.message())); - } - - T* operator->() const { return handle_; } - bool has_value() const { return handle_ != nullptr; } - T* get() const { return handle_; } - - static MetadataHandle TestOnlyWrap(T* p) { return MetadataHandle(p); } - - private: - friend class promise_filter_detail::BaseCallData; - - explicit MetadataHandle(T* handle) : handle_(handle) {} - T* Unwrap() { - T* result = handle_; - handle_ = nullptr; - return result; - } - - T* handle_ = nullptr; -}; - -// Server metadata type -// TODO(ctiller): This should be a bespoke instance of MetadataMap<> -using ServerMetadata = grpc_metadata_batch; -using ServerMetadataHandle = MetadataHandle; - -// Ok/not-ok check for trailing metadata, so that it can be used as result types -// for TrySeq. -inline bool IsStatusOk(const ServerMetadataHandle& m) { - return m->get(GrpcStatusMetadata()).value_or(GRPC_STATUS_UNKNOWN) == - GRPC_STATUS_OK; -} - -// Client initial metadata type -// TODO(ctiller): This should be a bespoke instance of MetadataMap<> -using ClientMetadata = grpc_metadata_batch; -using ClientMetadataHandle = MetadataHandle; - -// Server initial metadata type -// TODO(ctiller): This should be a bespoke instance of MetadataMap<> -using ServerMetadataHandle = MetadataHandle; struct CallArgs { + // Initial metadata from the client to the server. + // During promise setup this can be manipulated by filters (and then + // passed on to the next filter). ClientMetadataHandle client_initial_metadata; + // Initial metadata from the server to the client. + // Set once when it's available. + // During promise setup filters can substitute their own latch for this + // and consequently intercept the sent value and mutate/observe it. Latch* server_initial_metadata; + // Messages travelling from the application to the transport. + PipeReceiver* outgoing_messages; + // Messages travelling from the transport to the application. + PipeSender* incoming_messages; }; using NextPromiseFactory = @@ -538,6 +475,10 @@ void grpc_transport_stream_op_batch_finish_with_failure( void grpc_transport_stream_op_batch_queue_finish_with_failure( grpc_transport_stream_op_batch* batch, grpc_error_handle error, grpc_core::CallCombinerClosureList* closures); +// Fail a batch from within the transport (i.e. without the activity lock/call +// combiner taken). +void grpc_transport_stream_op_batch_finish_with_failure_from_transport( + grpc_transport_stream_op_batch* batch, grpc_error_handle error); std::string grpc_transport_stream_op_batch_string( grpc_transport_stream_op_batch* op); diff --git a/src/core/lib/transport/transport_impl.h b/src/core/lib/transport/transport_impl.h index 0d9a040fad3..44e7c46ab86 100644 --- a/src/core/lib/transport/transport_impl.h +++ b/src/core/lib/transport/transport_impl.h @@ -30,6 +30,7 @@ #include "src/core/lib/iomgr/iomgr_fwd.h" #include "src/core/lib/promise/arena_promise.h" #include "src/core/lib/resource_quota/arena.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/transport.h" #include "src/core/lib/transport/transport_fwd.h" @@ -56,7 +57,7 @@ typedef struct grpc_transport_vtable { There is an on-going migration to move all filters to providing this, and then to drop perform_stream_op. */ grpc_core::ArenaPromise (*make_call_promise)( - grpc_transport* self, grpc_core::ClientMetadataHandle initial_metadata); + grpc_transport* self, grpc_core::CallArgs call_args); /* implementation of grpc_transport_set_pollset */ void (*set_pollset)(grpc_transport* self, grpc_stream* stream, diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index cfd9f81f584..ee82bac8df8 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -685,6 +685,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/surface/call.cc', 'src/core/lib/surface/call_details.cc', 'src/core/lib/surface/call_log_batch.cc', + 'src/core/lib/surface/call_trace.cc', 'src/core/lib/surface/channel.cc', 'src/core/lib/surface/channel_init.cc', 'src/core/lib/surface/channel_ping.cc', @@ -700,6 +701,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/surface/validate_metadata.cc', 'src/core/lib/surface/version.cc', 'src/core/lib/transport/bdp_estimator.cc', + 'src/core/lib/transport/call_fragments.cc', 'src/core/lib/transport/connectivity_state.cc', 'src/core/lib/transport/error_utils.cc', 'src/core/lib/transport/handshaker.cc', diff --git a/test/core/end2end/end2end_tests.h b/test/core/end2end/end2end_tests.h index e77b1253bb6..50388971c05 100644 --- a/test/core/end2end/end2end_tests.h +++ b/test/core/end2end/end2end_tests.h @@ -46,6 +46,7 @@ typedef struct grpc_end2end_test_config grpc_end2end_test_config; #define FEATURE_MASK_DOES_NOT_SUPPORT_NETWORK_STATUS_CHANGE 256 #define FEATURE_MASK_SUPPORTS_WORKAROUNDS 512 #define FEATURE_MASK_DOES_NOT_SUPPORT_CLIENT_HANDSHAKE_COMPLETE_FIRST 1024 +#define FEATURE_MASK_DOES_NOT_SUPPORT_DEADLINES 2048 #define FAIL_AUTH_CHECK_SERVER_ARG_NAME "fail_auth_check" diff --git a/test/core/end2end/fixtures/h2_http_proxy.cc b/test/core/end2end/fixtures/h2_http_proxy.cc index c647aa4f40b..7646cb012c8 100644 --- a/test/core/end2end/fixtures/h2_http_proxy.cc +++ b/test/core/end2end/fixtures/h2_http_proxy.cc @@ -27,7 +27,6 @@ #include #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/gprpp/env.h" #include "src/core/lib/gprpp/host_port.h" #include "test/core/end2end/end2end_tests.h" #include "test/core/end2end/fixtures/http_proxy_fixture.h" @@ -75,9 +74,12 @@ void chttp2_init_client_fullstack(grpc_end2end_test_fixture* f, absl::StrFormat("http://%s@%s", proxy_auth_str, grpc_end2end_http_proxy_get_proxy_name(ffd->proxy)); } - grpc_core::SetEnv("http_proxy", proxy_uri.c_str()); grpc_channel_credentials* creds = grpc_insecure_credentials_create(); - f->client = grpc_channel_create(ffd->server_addr.c_str(), creds, client_args); + f->client = grpc_channel_create(ffd->server_addr.c_str(), creds, + grpc_core::ChannelArgs::FromC(client_args) + .Set(GRPC_ARG_HTTP_PROXY, proxy_uri) + .ToC() + .get()); grpc_channel_credentials_release(creds); GPR_ASSERT(f->client); } diff --git a/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc b/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc new file mode 100644 index 00000000000..d200338eb82 --- /dev/null +++ b/test/core/end2end/fixtures/h2_sockpair_with_minstack.cc @@ -0,0 +1,176 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" + +#include +#include +#include +#include + +#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_args_preconditioning.h" +#include "src/core/lib/channel/channelz.h" +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/endpoint.h" +#include "src/core/lib/iomgr/endpoint_pair.h" +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/surface/completion_queue.h" +#include "src/core/lib/surface/server.h" +#include "src/core/lib/transport/transport.h" +#include "src/core/lib/transport/transport_fwd.h" +#include "test/core/end2end/end2end_tests.h" +#include "test/core/util/test_config.h" + +/* chttp2 transport that is immediately available (used for testing + connected_channel without a client_channel) */ + +struct custom_fixture_data { + grpc_endpoint_pair ep; +}; + +static void server_setup_transport(void* ts, grpc_transport* transport) { + grpc_end2end_test_fixture* f = static_cast(ts); + grpc_core::ExecCtx exec_ctx; + custom_fixture_data* fixture_data = + static_cast(f->fixture_data); + grpc_endpoint_add_to_pollset(fixture_data->ep.server, grpc_cq_pollset(f->cq)); + grpc_core::Server* core_server = grpc_core::Server::FromC(f->server); + grpc_error_handle error = core_server->SetupTransport( + transport, nullptr, core_server->channel_args(), nullptr); + if (error == GRPC_ERROR_NONE) { + grpc_chttp2_transport_start_reading(transport, nullptr, nullptr, nullptr); + } else { + GRPC_ERROR_UNREF(error); + grpc_transport_destroy(transport); + } +} + +typedef struct { + grpc_end2end_test_fixture* f; + const grpc_channel_args* client_args; +} sp_client_setup; + +static void client_setup_transport(void* ts, grpc_transport* transport) { + sp_client_setup* cs = static_cast(ts); + + auto args = grpc_core::ChannelArgs::FromC(cs->client_args) + .Set(GRPC_ARG_MINIMAL_STACK, true) + .Set(GRPC_ARG_DEFAULT_AUTHORITY, "test-authority"); + auto channel = grpc_core::Channel::Create( + "socketpair-target", args, GRPC_CLIENT_DIRECT_CHANNEL, transport); + if (channel.ok()) { + cs->f->client = channel->release()->c_ptr(); + grpc_chttp2_transport_start_reading(transport, nullptr, nullptr, nullptr); + } else { + cs->f->client = grpc_lame_client_channel_create( + nullptr, static_cast(channel.status().code()), + "lame channel"); + grpc_transport_destroy(transport); + } +} + +static grpc_end2end_test_fixture chttp2_create_fixture_socketpair( + const grpc_channel_args* /*client_args*/, + const grpc_channel_args* /*server_args*/) { + custom_fixture_data* fixture_data = static_cast( + gpr_malloc(sizeof(custom_fixture_data))); + grpc_end2end_test_fixture f; + memset(&f, 0, sizeof(f)); + f.fixture_data = fixture_data; + f.cq = grpc_completion_queue_create_for_next(nullptr); + fixture_data->ep = grpc_iomgr_create_endpoint_pair("fixture", nullptr); + return f; +} + +static void chttp2_init_client_socketpair( + grpc_end2end_test_fixture* f, const grpc_channel_args* client_args) { + grpc_core::ExecCtx exec_ctx; + auto* fixture_data = static_cast(f->fixture_data); + grpc_transport* transport; + sp_client_setup cs; + auto final_client_args = grpc_core::CoreConfiguration::Get() + .channel_args_preconditioning() + .PreconditionChannelArgs(client_args) + .Set(GRPC_ARG_MINIMAL_STACK, true); + auto c_client_args = final_client_args.ToC(); + cs.client_args = c_client_args.get(); + cs.f = f; + transport = grpc_create_chttp2_transport(final_client_args, + fixture_data->ep.client, true); + client_setup_transport(&cs, transport); + GPR_ASSERT(f->client); +} + +static void chttp2_init_server_socketpair( + grpc_end2end_test_fixture* f, const grpc_channel_args* server_args) { + grpc_core::ExecCtx exec_ctx; + auto* fixture_data = static_cast(f->fixture_data); + grpc_transport* transport; + GPR_ASSERT(!f->server); + f->server = grpc_server_create(server_args, nullptr); + grpc_server_register_completion_queue(f->server, f->cq, nullptr); + grpc_server_start(f->server); + auto final_server_args = grpc_core::CoreConfiguration::Get() + .channel_args_preconditioning() + .PreconditionChannelArgs(server_args) + .Set(GRPC_ARG_MINIMAL_STACK, true); + transport = grpc_create_chttp2_transport(final_server_args, + fixture_data->ep.server, false); + server_setup_transport(f, transport); +} + +static void chttp2_tear_down_socketpair(grpc_end2end_test_fixture* f) { + grpc_core::ExecCtx exec_ctx; + gpr_free(f->fixture_data); +} + +/* All test configurations */ +static grpc_end2end_test_config configs[] = { + {"chttp2/socketpair+minstack", + FEATURE_MASK_SUPPORTS_AUTHORITY_HEADER | + FEATURE_MASK_DOES_NOT_SUPPORT_DEADLINES, + nullptr, chttp2_create_fixture_socketpair, chttp2_init_client_socketpair, + chttp2_init_server_socketpair, chttp2_tear_down_socketpair}, +}; + +int main(int argc, char** argv) { + size_t i; + + grpc::testing::TestEnvironment env(&argc, argv); + grpc_end2end_tests_pre_init(); + grpc_init(); + + for (i = 0; i < sizeof(configs) / sizeof(*configs); i++) { + grpc_end2end_tests(argc, argv, configs[i]); + } + + grpc_shutdown(); + + return 0; +} diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index 0671601d658..e8e23889076 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -33,6 +33,7 @@ def _fixture_options( is_inproc = False, is_1byte = False, is_http2 = True, + is_minstack = False, supports_proxy_auth = False, supports_write_buffering = True, client_channel = True, @@ -51,6 +52,7 @@ def _fixture_options( is_inproc = is_inproc, is_1byte = is_1byte, is_http2 = is_http2, + is_minstack = is_minstack, supports_proxy_auth = supports_proxy_auth, supports_write_buffering = supports_write_buffering, client_channel = client_channel, @@ -95,6 +97,12 @@ END2END_FIXTURES = { dns_resolver = False, client_channel = False, ), + "h2_sockpair_with_minstack": _fixture_options( + fullstack = False, + dns_resolver = False, + client_channel = False, + is_minstack = True, + ), "h2_sockpair+trace": _fixture_options( fullstack = False, dns_resolver = False, @@ -165,6 +173,7 @@ def _test_options( traceable = False, exclude_inproc = False, exclude_1byte = False, + exclude_minstack = False, needs_http2 = False, needs_proxy_auth = False, needs_write_buffering = False, @@ -182,6 +191,7 @@ def _test_options( traceable = traceable, exclude_inproc = exclude_inproc, exclude_1byte = exclude_1byte, + exclude_minstack = exclude_minstack, needs_http2 = needs_http2, needs_proxy_auth = needs_proxy_auth, needs_write_buffering = needs_write_buffering, @@ -217,7 +227,7 @@ END2END_TESTS = { "cancel_in_a_vacuum": _test_options(), "cancel_with_status": _test_options(), "client_streaming": _test_options(), - "compressed_payload": _test_options(proxyable = False, exclude_inproc = True), + "compressed_payload": _test_options(proxyable = False, exclude_inproc = True, exclude_minstack = True), "connectivity": _test_options( needs_fullstack = True, needs_names = True, @@ -249,11 +259,12 @@ END2END_TESTS = { "max_concurrent_streams": _test_options( proxyable = False, exclude_inproc = True, + exclude_minstack = True, ), "max_connection_age": _test_options(exclude_inproc = True), "max_connection_idle": _test_options(needs_fullstack = True, proxyable = False), - "max_message_length": _test_options(), - "negative_deadline": _test_options(), + "max_message_length": _test_options(exclude_minstack = True), + "negative_deadline": _test_options(exclude_minstack = True), "no_error_on_hotpath": _test_options(proxyable = False), "no_logging": _test_options(traceable = False), "no_op": _test_options(), @@ -389,6 +400,9 @@ def _compatible(fopt, topt): if topt.exclude_1byte: if fopt.is_1byte: return False + if topt.exclude_minstack: + if fopt.is_minstack: + return False if topt.needs_http2: if not fopt.is_http2: return False diff --git a/test/core/end2end/tests/cancel_after_accept.cc b/test/core/end2end/tests/cancel_after_accept.cc index 8d1f502e3b0..0143a8ab969 100644 --- a/test/core/end2end/tests/cancel_after_accept.cc +++ b/test/core/end2end/tests/cancel_after_accept.cc @@ -257,6 +257,10 @@ void cancel_after_accept(grpc_end2end_test_config config) { unsigned i; for (i = 0; i < GPR_ARRAY_SIZE(cancellation_modes); i++) { + if (config.feature_mask & FEATURE_MASK_DOES_NOT_SUPPORT_DEADLINES && + cancellation_modes[i].expect_status == GRPC_STATUS_DEADLINE_EXCEEDED) { + continue; + } test_cancel_after_accept(config, cancellation_modes[i], false /* use_service_config */); if (config.feature_mask & FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL && diff --git a/test/core/end2end/tests/cancel_after_client_done.cc b/test/core/end2end/tests/cancel_after_client_done.cc index 1e5e266f319..0471f2988fd 100644 --- a/test/core/end2end/tests/cancel_after_client_done.cc +++ b/test/core/end2end/tests/cancel_after_client_done.cc @@ -231,6 +231,10 @@ void cancel_after_client_done(grpc_end2end_test_config config) { unsigned i; for (i = 0; i < GPR_ARRAY_SIZE(cancellation_modes); i++) { + if (config.feature_mask & FEATURE_MASK_DOES_NOT_SUPPORT_DEADLINES && + cancellation_modes[i].expect_status == GRPC_STATUS_DEADLINE_EXCEEDED) { + continue; + } test_cancel_after_accept_and_writes_closed(config, cancellation_modes[i]); } } diff --git a/test/core/end2end/tests/cancel_after_invoke.cc b/test/core/end2end/tests/cancel_after_invoke.cc index 061d28b262a..37433ba876a 100644 --- a/test/core/end2end/tests/cancel_after_invoke.cc +++ b/test/core/end2end/tests/cancel_after_invoke.cc @@ -187,6 +187,11 @@ void cancel_after_invoke(grpc_end2end_test_config config) { for (j = 3; j < 6; j++) { for (i = 0; i < GPR_ARRAY_SIZE(cancellation_modes); i++) { + if (config.feature_mask & FEATURE_MASK_DOES_NOT_SUPPORT_DEADLINES && + cancellation_modes[i].expect_status == + GRPC_STATUS_DEADLINE_EXCEEDED) { + continue; + } test_cancel_after_invoke(config, cancellation_modes[i], j); } } diff --git a/test/core/end2end/tests/cancel_after_round_trip.cc b/test/core/end2end/tests/cancel_after_round_trip.cc index 0f24c13e44e..8d5c5b3177f 100644 --- a/test/core/end2end/tests/cancel_after_round_trip.cc +++ b/test/core/end2end/tests/cancel_after_round_trip.cc @@ -292,6 +292,10 @@ void cancel_after_round_trip(grpc_end2end_test_config config) { unsigned i; for (i = 0; i < GPR_ARRAY_SIZE(cancellation_modes); i++) { + if (config.feature_mask & FEATURE_MASK_DOES_NOT_SUPPORT_DEADLINES && + cancellation_modes[i].expect_status == GRPC_STATUS_DEADLINE_EXCEEDED) { + continue; + } test_cancel_after_round_trip(config, cancellation_modes[i], false /* use_service_config */); if (config.feature_mask & FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL && diff --git a/test/core/end2end/tests/cancel_before_invoke.cc b/test/core/end2end/tests/cancel_before_invoke.cc index a3eec56ad4d..2efa72a48fe 100644 --- a/test/core/end2end/tests/cancel_before_invoke.cc +++ b/test/core/end2end/tests/cancel_before_invoke.cc @@ -160,7 +160,11 @@ static void test_cancel_before_invoke(grpc_end2end_test_config config, error = grpc_call_start_batch(c, ops, test_ops, tag(1), nullptr); GPR_ASSERT(GRPC_CALL_OK == error); - cqv.Expect(tag(1), true); + // Filter based stack tracks this as a failed op, promise based stack tracks + // it as a successful one with a failed request. The latter probably makes + // more sense, but since we can't tell from outside which case we have we + // accept either. + cqv.Expect(tag(1), grpc_core::CqVerifier::AnyStatus()); cqv.Verify(); GPR_ASSERT(status == GRPC_STATUS_CANCELLED); diff --git a/test/core/end2end/tests/cancel_with_status.cc b/test/core/end2end/tests/cancel_with_status.cc index 391c2f7bed8..132df510257 100644 --- a/test/core/end2end/tests/cancel_with_status.cc +++ b/test/core/end2end/tests/cancel_with_status.cc @@ -20,6 +20,8 @@ #include #include +#include + #include #include #include @@ -28,6 +30,7 @@ #include #include +#include "src/core/lib/surface/event_string.h" #include "test/core/end2end/cq_verifier.h" #include "test/core/end2end/end2end_tests.h" #include "test/core/util/test_config.h" @@ -68,6 +71,7 @@ static void shutdown_server(grpc_end2end_test_fixture* f) { grpc_server_shutdown_and_notify(f->server, f->cq, tag(1000)); grpc_event ev = grpc_completion_queue_next( f->cq, grpc_timeout_seconds_to_deadline(5), nullptr); + gpr_log(GPR_DEBUG, "shutdown event: %s", grpc_event_string(&ev).c_str()); GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); GPR_ASSERT(ev.tag == tag(1000)); grpc_server_destroy(f->server); diff --git a/test/core/end2end/tests/request_with_flags.cc b/test/core/end2end/tests/request_with_flags.cc index bbbe93a8004..84e9aca3159 100644 --- a/test/core/end2end/tests/request_with_flags.cc +++ b/test/core/end2end/tests/request_with_flags.cc @@ -19,6 +19,10 @@ #include #include +#include + +#include "absl/strings/str_cat.h" + #include #include #include @@ -94,8 +98,18 @@ static void test_invoke_request_with_flags( grpc_slice_from_copied_string("hello world"); grpc_byte_buffer* request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - grpc_end2end_test_fixture f = - begin_test(config, "test_invoke_request_with_flags", nullptr, nullptr); + grpc_end2end_test_fixture f = begin_test( + config, + absl::StrCat("test_invoke_request_with_flags[", + absl::Hex(flags_for_op[GRPC_OP_SEND_INITIAL_METADATA]), ",", + absl::Hex(flags_for_op[GRPC_OP_SEND_MESSAGE]), ",", + absl::Hex(flags_for_op[GRPC_OP_SEND_CLOSE_FROM_CLIENT]), ",", + absl::Hex(flags_for_op[GRPC_OP_RECV_INITIAL_METADATA]), ",", + absl::Hex(flags_for_op[GRPC_OP_RECV_STATUS_ON_CLIENT]), + "]=>", + grpc_call_error_to_string(call_start_batch_expected_result)) + .c_str(), + nullptr, nullptr); grpc_core::CqVerifier cqv(f.cq); grpc_op ops[6]; grpc_op* op; @@ -154,6 +168,9 @@ static void test_invoke_request_with_flags( GPR_ASSERT(expectation == error); if (expectation == GRPC_CALL_OK) { + if (config.feature_mask & FEATURE_MASK_DOES_NOT_SUPPORT_DEADLINES) { + GPR_ASSERT(GRPC_CALL_OK == grpc_call_cancel(c, nullptr)); + } cqv.Expect(tag(1), true); cqv.Verify(); grpc_slice_unref(details); diff --git a/test/core/end2end/tests/simple_request.cc b/test/core/end2end/tests/simple_request.cc index 8b328ccb628..d861f09e3fd 100644 --- a/test/core/end2end/tests/simple_request.cc +++ b/test/core/end2end/tests/simple_request.cc @@ -41,6 +41,7 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, grpc_channel_args* client_args, grpc_channel_args* server_args) { grpc_end2end_test_fixture f; + gpr_log(GPR_INFO, "%s", std::string(100, '*').c_str()); gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name); f = config.create_fixture(client_args, server_args); config.init_server(&f, server_args); diff --git a/test/core/end2end/tests/streaming_error_response.cc b/test/core/end2end/tests/streaming_error_response.cc index e69a7c1f995..562041a7699 100644 --- a/test/core/end2end/tests/streaming_error_response.cc +++ b/test/core/end2end/tests/streaming_error_response.cc @@ -225,6 +225,8 @@ static void test(grpc_end2end_test_config config, bool request_status_early, cqv.Expect(tag(2), true); cqv.Verify(); + + GPR_ASSERT(response_payload2_recv != nullptr); } // Cancel the call so that the client sets up an error status. diff --git a/test/core/filters/client_auth_filter_test.cc b/test/core/filters/client_auth_filter_test.cc index 8424fed487f..6bfcb5a563e 100644 --- a/test/core/filters/client_auth_filter_test.cc +++ b/test/core/filters/client_auth_filter_test.cc @@ -49,6 +49,7 @@ #include "src/core/lib/security/security_connector/security_connector.h" #include "src/core/lib/security/transport/auth_filters.h" #include "src/core/lib/slice/slice.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" #include "test/core/promise/test_context.h" @@ -153,10 +154,8 @@ TEST_F(ClientAuthFilterTest, CallCredsFails) { TestContext context(arena_.get()); TestContext promise_call_context(call_context_); auto promise = filter->MakeCallPromise( - CallArgs{ - ClientMetadataHandle::TestOnlyWrap(&initial_metadata_batch_), - nullptr, - }, + CallArgs{ClientMetadataHandle::TestOnlyWrap(&initial_metadata_batch_), + nullptr, nullptr, nullptr}, [&](CallArgs /*call_args*/) { return ArenaPromise( [&]() -> Poll { @@ -185,10 +184,8 @@ TEST_F(ClientAuthFilterTest, RewritesInvalidStatusFromCallCreds) { TestContext context(arena_.get()); TestContext promise_call_context(call_context_); auto promise = filter->MakeCallPromise( - CallArgs{ - ClientMetadataHandle::TestOnlyWrap(&initial_metadata_batch_), - nullptr, - }, + CallArgs{ClientMetadataHandle::TestOnlyWrap(&initial_metadata_batch_), + nullptr, nullptr, nullptr}, [&](CallArgs /*call_args*/) { return ArenaPromise( [&]() -> Poll { diff --git a/test/core/filters/client_authority_filter_test.cc b/test/core/filters/client_authority_filter_test.cc index 3c20e716791..01508a72d4a 100644 --- a/test/core/filters/client_authority_filter_test.cc +++ b/test/core/filters/client_authority_filter_test.cc @@ -71,10 +71,8 @@ TEST(ClientAuthorityFilterTest, PromiseCompletesImmediatelyAndSetsAuthority) { // TODO(ctiller): use Activity here, once it's ready. TestContext context(arena.get()); auto promise = filter.MakeCallPromise( - CallArgs{ - ClientMetadataHandle::TestOnlyWrap(&initial_metadata_batch), - nullptr, - }, + CallArgs{ClientMetadataHandle::TestOnlyWrap(&initial_metadata_batch), + nullptr, nullptr, nullptr}, [&](CallArgs call_args) { EXPECT_EQ(call_args.client_initial_metadata ->get_pointer(HttpAuthorityMetadata()) @@ -105,10 +103,8 @@ TEST(ClientAuthorityFilterTest, // TODO(ctiller): use Activity here, once it's ready. TestContext context(arena.get()); auto promise = filter.MakeCallPromise( - CallArgs{ - ClientMetadataHandle::TestOnlyWrap(&initial_metadata_batch), - nullptr, - }, + CallArgs{ClientMetadataHandle::TestOnlyWrap(&initial_metadata_batch), + nullptr, nullptr, nullptr}, [&](CallArgs call_args) { EXPECT_EQ(call_args.client_initial_metadata ->get_pointer(HttpAuthorityMetadata()) diff --git a/test/core/filters/filter_fuzzer.cc b/test/core/filters/filter_fuzzer.cc index f3dff25de59..52ec3b58c15 100644 --- a/test/core/filters/filter_fuzzer.cc +++ b/test/core/filters/filter_fuzzer.cc @@ -80,6 +80,7 @@ #include "src/core/lib/slice/slice.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/channel_stack_type.h" +#include "src/core/lib/transport/call_fragments.h" #include "src/core/lib/transport/handshaker.h" #include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/transport/transport.h" @@ -116,8 +117,9 @@ const grpc_transport_vtable kFakeTransportVTable = { [](grpc_transport*, grpc_stream*, grpc_stream_refcount*, const void*, Arena*) -> int { abort(); }, // make_call_promise - [](grpc_transport*, - ClientMetadataHandle) -> ArenaPromise { abort(); }, + [](grpc_transport*, CallArgs) -> ArenaPromise { + abort(); + }, // set_pollset [](grpc_transport*, grpc_stream*, grpc_pollset*) { abort(); }, // set_pollset_set @@ -317,11 +319,12 @@ const grpc_channel_filter* FindFilter(absl::string_view name) { class MainLoop { public: - MainLoop(RefCountedPtr channel_stack, + MainLoop(bool is_client, RefCountedPtr channel_stack, const ChannelArgs& channel_args) : memory_allocator_(channel_args.GetObject() ->memory_quota() ->CreateMemoryAllocator("test")), + is_client_(is_client), channel_stack_(std::move(channel_stack)) {} ~MainLoop() { @@ -349,9 +352,9 @@ class MainLoop { calls_.erase(action.call()); break; case filter_fuzzer::Action::kCreateCall: - calls_.emplace( - action.call(), - std::make_unique(this, action.call(), action.create_call())); + calls_.emplace(action.call(), std::make_unique( + this, action.call(), + action.create_call(), is_client_)); break; case filter_fuzzer::Action::kReceiveInitialMetadata: if (auto* call = GetCall(action.call())) { @@ -407,6 +410,10 @@ class MainLoop { } void Drop() override { delete this; } + std::string ActivityDebugTag() const override { + return "WakeCall(" + std::to_string(id_) + ")"; + } + private: MainLoop* const main_loop_; uint32_t id_; @@ -460,14 +467,20 @@ class MainLoop { }; Call(MainLoop* main_loop, uint32_t id, - const filter_fuzzer::Metadata& client_initial_metadata) + const filter_fuzzer::Metadata& client_initial_metadata, bool is_client) : main_loop_(main_loop), id_(id) { ScopedContext context(this); auto* server_initial_metadata = arena_->New>(); - promise_ = main_loop_->channel_stack_->MakeCallPromise( - CallArgs{std::move(*LoadMetadata(client_initial_metadata, - &client_initial_metadata_)), - server_initial_metadata}); + CallArgs call_args{std::move(*LoadMetadata(client_initial_metadata, + &client_initial_metadata_)), + server_initial_metadata, nullptr, nullptr}; + if (is_client) { + promise_ = main_loop_->channel_stack_->MakeClientCallPromise( + std::move(call_args)); + } else { + promise_ = main_loop_->channel_stack_->MakeServerCallPromise( + std::move(call_args)); + } Step(); } @@ -574,7 +587,7 @@ class MainLoop { }; template - absl::optional> LoadMetadata( + absl::optional> LoadMetadata( const filter_fuzzer::Metadata& metadata, std::unique_ptr* out) { if (*out != nullptr) return absl::nullopt; *out = std::make_unique(arena_.get()); @@ -582,7 +595,7 @@ class MainLoop { (*out)->Append(md.key(), Slice::FromCopiedString(md.value()), [](absl::string_view, const Slice&) {}); } - return MetadataHandle::TestOnlyWrap(out->get()); + return FragmentHandle::TestOnlyWrap(out->get()); } void Step() { @@ -626,6 +639,7 @@ class MainLoop { } MemoryAllocator memory_allocator_; + const bool is_client_; RefCountedPtr channel_stack_; std::map> calls_; std::vector wakeups_; @@ -681,7 +695,7 @@ DEFINE_PROTO_FUZZER(const filter_fuzzer::Msg& msg) { }(); if (stack.ok()) { - grpc_core::MainLoop main_loop(std::move(*stack), channel_args); + grpc_core::MainLoop main_loop(is_client, std::move(*stack), channel_args); for (const auto& action : msg.actions()) { grpc_timer_manager_tick(); main_loop.Run(action, &globals); diff --git a/test/core/promise/activity_test.cc b/test/core/promise/activity_test.cc index dfd62ed2f61..279ebbe9de6 100644 --- a/test/core/promise/activity_test.cc +++ b/test/core/promise/activity_test.cc @@ -344,6 +344,7 @@ class TestWakeable final : public Wakeable { drops_->fetch_add(1, std::memory_order_relaxed); delete this; } + std::string ActivityDebugTag() const override { return "TestWakeable"; } private: std::atomic* const wakeups_; diff --git a/test/core/promise/arena_promise_test.cc b/test/core/promise/arena_promise_test.cc index ab11f2fae60..b7c37900d80 100644 --- a/test/core/promise/arena_promise_test.cc +++ b/test/core/promise/arena_promise_test.cc @@ -33,12 +33,20 @@ namespace grpc_core { static auto* g_memory_allocator = new MemoryAllocator( ResourceQuota::Default()->memory_quota()->CreateMemoryAllocator("test")); +TEST(ArenaPromiseTest, DefaultInitializationYieldsNoValue) { + auto arena = MakeScopedArena(1024, g_memory_allocator); + TestContext context(arena.get()); + ArenaPromise p; + EXPECT_FALSE(p.has_value()); +} + TEST(ArenaPromiseTest, AllocatedWorks) { ExecCtx exec_ctx; auto arena = MakeScopedArena(1024, g_memory_allocator); TestContext context(arena.get()); int x = 42; ArenaPromise p([x] { return Poll(x); }); + EXPECT_TRUE(p.has_value()); EXPECT_EQ(p(), Poll(42)); p = ArenaPromise([] { return Poll(43); }); EXPECT_EQ(p(), Poll(43)); diff --git a/test/core/promise/call_push_pull_test.cc b/test/core/promise/call_push_pull_test.cc index a0464424621..89d2ab8107f 100644 --- a/test/core/promise/call_push_pull_test.cc +++ b/test/core/promise/call_push_pull_test.cc @@ -54,8 +54,8 @@ TEST(CallPushPullTest, OneReady) { TEST(CallPushPullTest, OneFailed) { auto a = CallPushPull( []() -> Poll { return absl::UnknownError("bah"); }, - []() -> Poll { return Pending{}; }, - []() -> Poll { return Pending{}; }); + []() -> Poll { return absl::OkStatus(); }, + []() -> Poll { return absl::OkStatus(); }); EXPECT_EQ(a(), Poll(absl::UnknownError("bah"))); auto b = CallPushPull( []() -> Poll { return Pending{}; }, diff --git a/test/core/surface/BUILD b/test/core/surface/BUILD index 61c6e27b969..a405836582f 100644 --- a/test/core/surface/BUILD +++ b/test/core/surface/BUILD @@ -89,6 +89,7 @@ grpc_cc_test( srcs = ["lame_client_test.cc"], external_deps = ["gtest"], language = "C++", + tags = ["lame_client_test"], deps = [ "//:gpr", "//:grpc", diff --git a/test/core/surface/lame_client_test.cc b/test/core/surface/lame_client_test.cc index 85db3ea19a0..8d3f3947986 100644 --- a/test/core/surface/lame_client_test.cc +++ b/test/core/surface/lame_client_test.cc @@ -30,6 +30,7 @@ #include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/experiments/experiments.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/iomgr/closure.h" #include "src/core/lib/iomgr/error.h" @@ -123,8 +124,11 @@ TEST(LameClientTest, MainTest) { tag(1), nullptr); ASSERT_EQ(GRPC_CALL_OK, error); - /* the call should immediately fail */ - cqv.Expect(tag(1), false); + // Filter stack code considers this a failed to receive initial metadata + // result, where as promise based code interprets this as a trailers only + // failed request. Both are rational interpretations, so we accept the one + // that is implemented for each stack. + cqv.Expect(tag(1), grpc_core::IsPromiseBasedClientCallEnabled()); cqv.Verify(); memset(ops, 0, sizeof(ops)); diff --git a/test/core/transport/BUILD b/test/core/transport/BUILD index 142e22b2c83..1550a2472ee 100644 --- a/test/core/transport/BUILD +++ b/test/core/transport/BUILD @@ -65,6 +65,21 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "call_fragments_test", + srcs = ["call_fragments_test.cc"], + external_deps = [ + "gtest", + ], + language = "C++", + deps = [ + "//:gpr", + "//:grpc", + "//test/core/promise:test_context", + "//test/core/util:grpc_test_util", + ], +) + grpc_cc_test( name = "metadata_map_test", srcs = ["metadata_map_test.cc"], diff --git a/test/core/transport/call_fragments_test.cc b/test/core/transport/call_fragments_test.cc new file mode 100644 index 00000000000..d38e3a466ba --- /dev/null +++ b/test/core/transport/call_fragments_test.cc @@ -0,0 +1,96 @@ +// +// Copyright 2021 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "src/core/lib/transport/call_fragments.h" + +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include + +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/resource_quota/memory_quota.h" +#include "src/core/lib/resource_quota/resource_quota.h" +#include "test/core/promise/test_context.h" +#include "test/core/util/test_config.h" + +using testing::Each; + +namespace grpc_core { +namespace testing { + +class CallFragmentsTest : public ::testing::Test { + protected: + CallFragmentsTest() {} + ~CallFragmentsTest() override {} + + private: + MemoryAllocator memory_allocator_ = + ResourceQuota::Default()->memory_quota()->CreateMemoryAllocator("test"); + ScopedArenaPtr arena_ = MakeScopedArena(4096, &memory_allocator_); + FragmentAllocator fragment_allocator_; + + TestContext arena_context_{arena_.get()}; + TestContext fragment_allocator_context_{ + &fragment_allocator_}; +}; + +// Ensure test fixture can init/destroy successfully. +TEST_F(CallFragmentsTest, Nothing) {} + +// Ensure we can create/destroy some client metadata. +TEST_F(CallFragmentsTest, ClientMetadata) { + GetContext()->MakeClientMetadata(); +} + +// Ensure we can create/destroy some server metadata. +TEST_F(CallFragmentsTest, ServerMetadata) { + GetContext()->MakeServerMetadata(); +} + +// Ensure repeated allocation/deallocations reuse memory. +TEST_F(CallFragmentsTest, RepeatedAllocationsReuseMemory) { + void* p = GetContext()->MakeClientMetadata().get(); + void* q = GetContext()->MakeClientMetadata().get(); + EXPECT_EQ(p, q); +} + +// Ensure repeated allocation reinitializes. +TEST_F(CallFragmentsTest, RepeatedAllocationsReinitialize) { + std::vector addresses; + for (int i = 0; i < 4; i++) { + ClientMetadataHandle metadata = + GetContext()->MakeClientMetadata(); + EXPECT_EQ(metadata->get_pointer(HttpPathMetadata()), nullptr); + metadata->Set(HttpPathMetadata(), Slice::FromCopiedString("/")); + EXPECT_EQ(metadata->get_pointer(HttpPathMetadata())->as_string_view(), "/"); + addresses.push_back(metadata.get()); + } + EXPECT_THAT(addresses, Each(addresses[0])); +} + +} // namespace testing +} // namespace grpc_core + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + grpc::testing::TestEnvironment env(&argc, argv); + return RUN_ALL_TESTS(); +}; diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 138d6c86158..2677665b8e5 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -2256,6 +2256,7 @@ src/core/lib/promise/intra_activity_waiter.h \ src/core/lib/promise/latch.h \ src/core/lib/promise/loop.h \ src/core/lib/promise/map.h \ +src/core/lib/promise/pipe.h \ src/core/lib/promise/poll.h \ src/core/lib/promise/promise.h \ src/core/lib/promise/race.h \ @@ -2432,6 +2433,8 @@ src/core/lib/surface/call.h \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ src/core/lib/surface/call_test_only.h \ +src/core/lib/surface/call_trace.cc \ +src/core/lib/surface/call_trace.h \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel.h \ src/core/lib/surface/channel_init.cc \ @@ -2459,6 +2462,8 @@ src/core/lib/surface/validate_metadata.h \ src/core/lib/surface/version.cc \ src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.h \ +src/core/lib/transport/call_fragments.cc \ +src/core/lib/transport/call_fragments.h \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.h \ src/core/lib/transport/error_utils.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index ae5aad66a33..bd2e886d45e 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -2047,6 +2047,7 @@ src/core/lib/promise/intra_activity_waiter.h \ src/core/lib/promise/latch.h \ src/core/lib/promise/loop.h \ src/core/lib/promise/map.h \ +src/core/lib/promise/pipe.h \ src/core/lib/promise/poll.h \ src/core/lib/promise/promise.h \ src/core/lib/promise/race.h \ @@ -2224,6 +2225,8 @@ src/core/lib/surface/call.h \ src/core/lib/surface/call_details.cc \ src/core/lib/surface/call_log_batch.cc \ src/core/lib/surface/call_test_only.h \ +src/core/lib/surface/call_trace.cc \ +src/core/lib/surface/call_trace.h \ src/core/lib/surface/channel.cc \ src/core/lib/surface/channel.h \ src/core/lib/surface/channel_init.cc \ @@ -2252,6 +2255,8 @@ src/core/lib/surface/version.cc \ src/core/lib/transport/README.md \ src/core/lib/transport/bdp_estimator.cc \ src/core/lib/transport/bdp_estimator.h \ +src/core/lib/transport/call_fragments.cc \ +src/core/lib/transport/call_fragments.h \ src/core/lib/transport/connectivity_state.cc \ src/core/lib/transport/connectivity_state.h \ src/core/lib/transport/error_utils.cc \ diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 801a6a021ed..2d803570d53 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1415,6 +1415,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "call_fragments_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false,