[chaotic-good] Multi-connection support (#38032)

Builds upon #37765 to support arbitrary connection counts in the transport.
(note: at this point the number of connections is determined at connection establishment - future work will be autotuning this)

Closes #38032

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/38032 from ctiller:tiefling-buffer c7520fd7a9
PiperOrigin-RevId: 698952890

CMakeLists.txt

@@ -8502,6 +8502,8 @@ add_executable(bad_ping_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -8989,6 +8991,8 @@ add_executable(binary_metadata_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -9334,6 +9338,8 @@ add_executable(call_creds_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -9546,6 +9552,8 @@ add_executable(call_host_override_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -10063,6 +10071,8 @@ add_executable(cancel_after_accept_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -10137,6 +10147,8 @@ add_executable(cancel_after_client_done_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -10211,6 +10223,8 @@ add_executable(cancel_after_invoke_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -10285,6 +10299,8 @@ add_executable(cancel_after_round_trip_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -10406,6 +10422,8 @@ add_executable(cancel_before_invoke_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -10557,6 +10575,8 @@ add_executable(cancel_in_a_vacuum_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -10631,6 +10651,8 @@ add_executable(cancel_with_status_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -12462,6 +12484,8 @@ add_executable(client_streaming_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -12812,6 +12836,8 @@ add_executable(compressed_payload_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -13212,6 +13238,8 @@ add_executable(connectivity_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -13684,6 +13712,8 @@ add_executable(default_host_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -13919,6 +13949,8 @@ add_executable(disappearing_server_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -14286,6 +14318,8 @@ add_executable(empty_batch_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -15437,6 +15471,8 @@ add_executable(filter_causes_close_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -15511,6 +15547,8 @@ add_executable(filter_init_fails_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -15635,6 +15673,8 @@ add_executable(filtered_metadata_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -16587,6 +16627,8 @@ add_executable(graceful_server_shutdown_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -16980,6 +17022,8 @@ add_executable(grpc_authz_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -18431,6 +18475,8 @@ add_executable(high_initial_seqno_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -18751,6 +18797,8 @@ add_executable(hpack_size_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -19026,6 +19074,8 @@ add_executable(http2_stats_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -19986,6 +20036,8 @@ add_executable(invoke_large_request_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -20388,6 +20440,8 @@ add_executable(keepalive_timeout_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -20505,6 +20559,8 @@ add_executable(large_metadata_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -21283,6 +21339,8 @@ add_executable(max_concurrent_streams_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -21357,6 +21415,8 @@ add_executable(max_connection_age_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -21431,6 +21491,8 @@ add_executable(max_connection_idle_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -21505,6 +21567,8 @@ add_executable(max_message_length_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -22254,6 +22318,8 @@ add_executable(negative_deadline_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -22360,6 +22426,8 @@ add_executable(no_logging_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -22434,6 +22502,8 @@ add_executable(no_op_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -23474,6 +23544,8 @@ add_executable(payload_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -23852,6 +23924,8 @@ add_executable(ping_pong_streaming_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -23977,6 +24051,8 @@ add_executable(ping_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -25520,6 +25596,8 @@ add_executable(proxy_auth_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -26123,6 +26201,8 @@ add_executable(registered_call_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -26341,6 +26421,8 @@ add_executable(request_with_flags_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -26415,6 +26497,8 @@ add_executable(request_with_payload_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -26776,6 +26860,8 @@ add_executable(resource_quota_server_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -26892,6 +26978,8 @@ add_executable(retry_cancel_after_first_attempt_starts_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -26966,6 +27054,8 @@ add_executable(retry_cancel_during_delay_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27040,6 +27130,8 @@ add_executable(retry_cancel_with_multiple_send_batches_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27114,6 +27206,8 @@ add_executable(retry_cancellation_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27188,6 +27282,8 @@ add_executable(retry_disabled_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27262,6 +27358,8 @@ add_executable(retry_exceeds_buffer_size_in_delay_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27336,6 +27434,8 @@ add_executable(retry_exceeds_buffer_size_in_initial_batch_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27410,6 +27510,8 @@ add_executable(retry_exceeds_buffer_size_in_subsequent_batch_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27484,6 +27586,8 @@ add_executable(retry_lb_drop_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27558,6 +27662,8 @@ add_executable(retry_lb_fail_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27632,6 +27738,8 @@ add_executable(retry_non_retriable_status_before_trailers_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27706,6 +27814,8 @@ add_executable(retry_non_retriable_status_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27780,6 +27890,8 @@ add_executable(retry_per_attempt_recv_timeout_on_last_attempt_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27854,6 +27966,8 @@ add_executable(retry_per_attempt_recv_timeout_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -27928,6 +28042,8 @@ add_executable(retry_recv_initial_metadata_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28002,6 +28118,8 @@ add_executable(retry_recv_message_replay_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28076,6 +28194,8 @@ add_executable(retry_recv_message_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28150,6 +28270,8 @@ add_executable(retry_recv_trailing_metadata_error_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28224,6 +28346,8 @@ add_executable(retry_send_initial_metadata_refs_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28298,6 +28422,8 @@ add_executable(retry_send_op_fails_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28372,6 +28498,8 @@ add_executable(retry_send_recv_batch_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28446,6 +28574,8 @@ add_executable(retry_server_pushback_delay_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28520,6 +28650,8 @@ add_executable(retry_server_pushback_disabled_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28636,6 +28768,8 @@ add_executable(retry_streaming_after_commit_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28710,6 +28844,8 @@ add_executable(retry_streaming_succeeds_before_replay_finished_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28784,6 +28920,8 @@ add_executable(retry_streaming_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28858,6 +28996,8 @@ add_executable(retry_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -28974,6 +29114,8 @@ add_executable(retry_throttled_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -29048,6 +29190,8 @@ add_executable(retry_too_many_attempts_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -29122,6 +29266,8 @@ add_executable(retry_transparent_goaway_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -29196,6 +29342,8 @@ add_executable(retry_transparent_max_concurrent_streams_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -29270,6 +29418,8 @@ add_executable(retry_transparent_not_sent_on_wire_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -29344,6 +29494,8 @@ add_executable(retry_unref_before_finish_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -29418,6 +29570,8 @@ add_executable(retry_unref_before_recv_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -30584,6 +30738,8 @@ add_executable(server_finishes_request_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -30911,6 +31067,8 @@ add_executable(server_streaming_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -31200,6 +31358,8 @@ add_executable(shutdown_finishes_calls_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -31274,6 +31434,8 @@ add_executable(shutdown_finishes_tags_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -31427,6 +31589,8 @@ add_executable(simple_delayed_request_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -31501,6 +31665,8 @@ add_executable(simple_metadata_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -31619,6 +31785,8 @@ add_executable(simple_request_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -32558,6 +32726,8 @@ add_executable(streaming_error_response_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -33361,6 +33531,8 @@ add_executable(test_core_end2end_channelz_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -33831,6 +34003,8 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX)
     ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h
     ${_gRPC_PROTO_GENS_DIR}/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.grpc.pb.h
     src/core/ext/transport/chaotic_good/client_transport.cc
+    src/core/ext/transport/chaotic_good/control_endpoint.cc
+    src/core/ext/transport/chaotic_good/data_endpoints.cc
     src/core/ext/transport/chaotic_good/frame.cc
     src/core/ext/transport/chaotic_good/frame_header.cc
     src/core/ext/transport/chaotic_good/server_transport.cc
@@ -34058,6 +34232,8 @@ add_executable(test_cpp_ext_chaotic_good_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -34644,6 +34820,8 @@ add_executable(timeout_before_request_call_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -35233,6 +35411,8 @@ add_executable(trailing_metadata_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -36214,6 +36394,8 @@ add_executable(write_buffering_at_end_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc
@@ -36288,6 +36470,8 @@ add_executable(write_buffering_test
   ${_gRPC_PROTO_GENS_DIR}/src/core/ext/transport/chaotic_good/chaotic_good_frame.grpc.pb.h
   src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc
   src/core/ext/transport/chaotic_good/client_transport.cc
+  src/core/ext/transport/chaotic_good/control_endpoint.cc
+  src/core/ext/transport/chaotic_good/data_endpoints.cc
   src/core/ext/transport/chaotic_good/frame.cc
   src/core/ext/transport/chaotic_good/frame_header.cc
   src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc

src/core/BUILD

@@ -928,6 +928,7 @@ grpc_cc_library(
     deps = [
         "join_state",
         "map",
+        "promise_factory",
         "//:gpr_platform",
     ],
 )
@@ -968,6 +969,7 @@ grpc_cc_library(
         "join_state",
         "map",
         "poll",
+        "promise_factory",
         "status_flag",
         "//:gpr_platform",
     ],
@@ -982,6 +984,7 @@ grpc_cc_library(
     deps = [
         "construct_destruct",
         "poll",
+        "promise_factory",
         "//:gpr_platform",
     ],
 )
@@ -1045,7 +1048,6 @@ grpc_cc_library(
         "promise_status",
         "seq_state",
         "status_flag",
-        "//:debug_location",
         "//:gpr_platform",
     ],
 )
@@ -8090,6 +8092,50 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "chaotic_good_control_endpoint",
+    srcs = [
+        "ext/transport/chaotic_good/control_endpoint.cc",
+    ],
+    hdrs = [
+        "ext/transport/chaotic_good/control_endpoint.h",
+    ],
+    external_deps = ["absl/cleanup"],
+    deps = [
+        "1999",
+        "event_engine_context",
+        "event_engine_tcp_socket_utils",
+        "grpc_promise_endpoint",
+        "loop",
+        "try_seq",
+        "//:gpr",
+    ],
+)
+
+grpc_cc_library(
+    name = "chaotic_good_data_endpoints",
+    srcs = [
+        "ext/transport/chaotic_good/data_endpoints.cc",
+    ],
+    hdrs = [
+        "ext/transport/chaotic_good/data_endpoints.h",
+    ],
+    external_deps = [
+        "absl/cleanup",
+        "absl/strings",
+    ],
+    deps = [
+        "1999",
+        "event_engine_context",
+        "grpc_promise_endpoint",
+        "loop",
+        "seq",
+        "slice_buffer",
+        "try_seq",
+        "//:promise",
+    ],
+)
+
 grpc_cc_library(
     name = "chaotic_good_transport",
     hdrs = [
@@ -8102,16 +8148,21 @@ grpc_cc_library(
     ],
     language = "c++",
     deps = [
+        "chaotic_good_control_endpoint",
+        "chaotic_good_data_endpoints",
         "chaotic_good_frame",
         "chaotic_good_frame_header",
+        "event_engine_context",
         "event_engine_tcp_socket_utils",
         "grpc_promise_endpoint",
-        "if",
+        "loop",
+        "match_promise",
+        "mpsc",
+        "seq",
         "try_join",
         "try_seq",
         "//:gpr_platform",
         "//:grpc_trace",
-        "//:promise",
     ],
 )
 
@@ -8126,8 +8177,8 @@ grpc_cc_library(
     external_deps = [
         "absl/base:core_headers",
         "absl/container:flat_hash_map",
-        "absl/log:check",
         "absl/log",
+        "absl/log:check",
         "absl/random",
         "absl/random:bit_gen_ref",
         "absl/status",
@@ -8184,8 +8235,8 @@ grpc_cc_library(
         "absl/base:core_headers",
         "absl/container:flat_hash_map",
         "absl/functional:any_invocable",
+        "absl/log",
         "absl/log:check",
-        "absl/log:log",
         "absl/random",
         "absl/random:bit_gen_ref",
         "absl/status",
@@ -8707,8 +8758,8 @@ grpc_cc_library(
     ],
     external_deps = [
         "absl/container:flat_hash_map",
+        "absl/log",
         "absl/log:check",
-        "absl/log:log",
         "absl/random",
         "absl/random:bit_gen_ref",
         "absl/status",
@@ -8739,6 +8790,7 @@ grpc_cc_library(
         "if",
         "inter_activity_latch",
         "iomgr_fwd",
+        "join",
         "latch",
         "memory_quota",
         "metadata",
@@ -8839,8 +8891,8 @@ grpc_cc_library(
         "ext/transport/chaotic_good/client/chaotic_good_connector.h",
     ],
     external_deps = [
+        "absl/log",
         "absl/log:check",
-        "absl/log:log",
         "absl/random",
         "absl/random:bit_gen_ref",
         "absl/status",
@@ -8849,6 +8901,7 @@ grpc_cc_library(
     language = "c++",
     deps = [
         "activity",
+        "all_ok",
         "arena",
         "channel_args",
         "channel_args_endpoint_config",

src/core/ext/transport/chaotic_good/chaotic_good_frame.proto

@@ -19,10 +19,10 @@ package chaotic_good_frame;
 message Settings {
     // Connection id
     // - sent server->client on the control channel to specify the
-    //   data channel connection id
+    //   data channel connection id, one for each desired connection
-    // - sent client->server on the data channel to complete the
+    // - exactly one sent client->server on the data channel to 
-    //   connection
+    //   complete the connection
-    bytes connection_id = 1;
+    repeated bytes connection_id = 1;
     // Flag true if this is a data channel (and not a control channel)
     bool data_channel = 2;
     // Requested alignment for the data channel

src/core/ext/transport/chaotic_good/chaotic_good_transport.h

@@ -20,15 +20,18 @@
 #include <cstdint>
 #include <utility>
 
-#include "absl/log/log.h"
-#include "absl/random/random.h"
 #include "absl/strings/escaping.h"
+#include "src/core/ext/transport/chaotic_good/control_endpoint.h"
+#include "src/core/ext/transport/chaotic_good/data_endpoints.h"
 #include "src/core/ext/transport/chaotic_good/frame.h"
 #include "src/core/ext/transport/chaotic_good/frame_header.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/event_engine/event_engine_context.h"
 #include "src/core/lib/event_engine/tcp_socket_utils.h"
-#include "src/core/lib/promise/if.h"
+#include "src/core/lib/promise/loop.h"
-#include "src/core/lib/promise/promise.h"
+#include "src/core/lib/promise/match_promise.h"
+#include "src/core/lib/promise/mpsc.h"
+#include "src/core/lib/promise/seq.h"
 #include "src/core/lib/promise/try_join.h"
 #include "src/core/lib/promise/try_seq.h"
 #include "src/core/lib/transport/promise_endpoint.h"
@@ -36,6 +39,49 @@
 namespace grpc_core {
 namespace chaotic_good {
 
+inline std::vector<PromiseEndpoint> OneDataEndpoint(PromiseEndpoint endpoint) {
+  std::vector<PromiseEndpoint> ep;
+  ep.emplace_back(std::move(endpoint));
+  return ep;
+}
+
+// One received frame: the header, and the serialized bytes of the payload.
+// The payload may not yet be received into memory, so the accessor for that
+// returns a promise that will need to be resolved prior to inspecting the
+// bytes.
+// In this way we can pull bytes from various different data connections and
+// read them in any order, but still have a trivial reassembly in the receiving
+// call promise.
+class IncomingFrame {
+ public:
+  template <typename T>
+  IncomingFrame(FrameHeader header, T payload, size_t remove_padding)
+      : header_(header),
+        payload_(std::move(payload)),
+        remove_padding_(remove_padding) {}
+
+  const FrameHeader& header() { return header_; }
+
+  auto Payload() {
+    return Map(
+        MatchPromise(
+            std::move(payload_),
+            [](absl::StatusOr<SliceBuffer> status) { return status; },
+            [](DataEndpoints::ReadTicket ticket) { return ticket.Await(); }),
+        [remove_padding =
+             remove_padding_](absl::StatusOr<SliceBuffer> payload) {
+          if (payload.ok()) payload->RemoveLastNBytesNoInline(remove_padding);
+          return payload;
+        });
+  }
+
+ private:
+  FrameHeader header_;
+  absl::variant<absl::StatusOr<SliceBuffer>, DataEndpoints::ReadTicket>
+      payload_;
+  size_t remove_padding_;
+};
+
 class ChaoticGoodTransport : public RefCounted<ChaoticGoodTransport> {
  public:
   struct Options {
@@ -44,47 +90,85 @@ class ChaoticGoodTransport : public RefCounted<ChaoticGoodTransport> {
     uint32_t inlined_payload_size_threshold = 8 * 1024;
   };
 
-  ChaoticGoodTransport(PromiseEndpoint control_endpoint,
+  ChaoticGoodTransport(
-                       PromiseEndpoint data_endpoint, Options options)
+      PromiseEndpoint control_endpoint,
-      : control_endpoint_(std::move(control_endpoint)),
+      std::vector<PromiseEndpoint> data_endpoints,
-        data_endpoint_(std::move(data_endpoint)),
+      std::shared_ptr<grpc_event_engine::experimental::EventEngine>
-        options_(options) {
+          event_engine,
-    // Enable RxMemoryAlignment and RPC receive coalescing after the transport
+      Options options)
-    // setup is complete. At this point all the settings frames should have
+      : event_engine_(std::move(event_engine)),
-    // been read.
+        control_endpoint_(std::move(control_endpoint), event_engine_.get()),
-    data_endpoint_.EnforceRxMemoryAlignmentAndCoalescing();
+        data_endpoints_(std::move(data_endpoints), event_engine_.get()),
-  }
+        options_(options) {}
 
   auto WriteFrame(const FrameInterface& frame) {
-    SliceBuffer control;
-    SliceBuffer data;
     FrameHeader header = frame.MakeHeader();
-    if (header.payload_length > options_.inlined_payload_size_threshold) {
-      header.payload_connection_id = 1;
-      header.Serialize(control.AddTiny(FrameHeader::kFrameHeaderSize));
-      frame.SerializePayload(data);
-      const size_t padding = header.Padding(options_.encode_alignment);
-      if (padding != 0) {
-        auto slice = MutableSlice::CreateUninitialized(padding);
-        memset(slice.data(), 0, padding);
-        data.AppendIndexed(Slice(std::move(slice)));
-      }
-    } else {
-      header.Serialize(control.AddTiny(FrameHeader::kFrameHeaderSize));
-      frame.SerializePayload(control);
-    }
-    // ignore encoding errors: they will be logged separately already
     GRPC_TRACE_LOG(chaotic_good, INFO)
         << "CHAOTIC_GOOD: WriteFrame to:"
         << ResolvedAddressToString(control_endpoint_.GetPeerAddress())
                .value_or("<<unknown peer address>>")
         << " " << frame.ToString();
-    return TryJoin<absl::StatusOr>(control_endpoint_.Write(std::move(control)),
+    return If(
-                                   data_endpoint_.Write(std::move(data)));
+        // If we have no data endpoints, OR this is a small payload
+        data_endpoints_.empty() ||
+            header.payload_length <= options_.inlined_payload_size_threshold,
+        // ... then write it to the control endpoint
+        [this, &header, &frame]() {
+          SliceBuffer output;
+          header.Serialize(output.AddTiny(FrameHeader::kFrameHeaderSize));
+          frame.SerializePayload(output);
+          return control_endpoint_.Write(std::move(output));
+        },
+        // ... otherwise write it to a data connection
+        [this, header, &frame]() mutable {
+          SliceBuffer payload;
+          // Temporarily give a bogus connection id to get padding right
+          header.payload_connection_id = 1;
+          const size_t padding = header.Padding(options_.encode_alignment);
+          frame.SerializePayload(payload);
+          GRPC_TRACE_LOG(chaotic_good, INFO)
+              << "CHAOTIC_GOOD: Send " << payload.Length()
+              << "b payload on data channel; add " << padding << " bytes for "
+              << options_.encode_alignment << " alignment";
+          if (padding != 0) {
+            auto slice = MutableSlice::CreateUninitialized(padding);
+            memset(slice.data(), 0, padding);
+            payload.AppendIndexed(Slice(std::move(slice)));
+          }
+          return Seq(data_endpoints_.Write(std::move(payload)),
+                     [this, header](uint32_t connection_id) mutable {
+                       header.payload_connection_id = connection_id + 1;
+                       SliceBuffer header_frame;
+                       header.Serialize(
+                           header_frame.AddTiny(FrameHeader::kFrameHeaderSize));
+                       return control_endpoint_.Write(std::move(header_frame));
+                     });
+        });
+  }
+
+  // Common outbound loop for both client and server (these vary only over the
+  // frame type).
+  template <typename Frame>
+  auto TransportWriteLoop(MpscReceiver<Frame>& outgoing_frames) {
+    return Loop([self = Ref(), &outgoing_frames] {
+      return TrySeq(
+          // Get next outgoing frame.
+          outgoing_frames.Next(),
+          // Serialize and write it out.
+          [self = self.get()](Frame client_frame) {
+            return self->WriteFrame(GetFrameInterface(client_frame));
+          },
+          []() -> LoopCtl<absl::Status> {
+            // The write failures will be caught in TrySeq and exit loop.
+            // Therefore, only need to return Continue() in the last lambda
+            // function.
+            return Continue();
+          });
+    });
   }
 
   // Read frame header and payloads for control and data portions of one frame.
-  // Resolves to StatusOr<tuple<FrameHeader, BufferPair>>.
+  // Resolves to StatusOr<IncomingFrame>.
   auto ReadFrameBytes() {
     return TrySeq(
         control_endpoint_.ReadSlice(FrameHeader::kFrameHeaderSize),
@@ -102,19 +186,36 @@ class ChaoticGoodTransport : public RefCounted<ChaoticGoodTransport> {
           return frame_header;
         },
         [this](FrameHeader frame_header) {
-          current_frame_header_ = frame_header;
+          return If(
-          auto con = frame_header.payload_connection_id == 0
+              // If the payload is on the connection frame
-                         ? &control_endpoint_
+              frame_header.payload_connection_id == 0,
-                         : &data_endpoint_;
+              // ... then read the data immediately and return an IncomingFrame
-          return con->Read(frame_header.payload_length +
+              //     that contains the payload.
-                           frame_header.Padding(options_.decode_alignment));
+              // We need to do this here so that we do not create head of line
-        },
+              // blocking issues reading later control frames (but waiting for a
-        [this](SliceBuffer payload)
+              // call to get scheduled time to read the payload).
-            -> absl::StatusOr<std::tuple<FrameHeader, SliceBuffer>> {
+              [this, frame_header]() {
-          payload.RemoveLastNBytesNoInline(
+                return Map(control_endpoint_.Read(frame_header.payload_length),
-              current_frame_header_.Padding(options_.decode_alignment));
+                           [frame_header](absl::StatusOr<SliceBuffer> payload)
-          return std::tuple<FrameHeader, SliceBuffer>(current_frame_header_,
+                               -> absl::StatusOr<IncomingFrame> {
-                                                      std::move(payload));
+                             if (!payload.ok()) return payload.status();
+                             return IncomingFrame(frame_header,
+                                                  std::move(payload), 0);
+                           });
+              },
+              // ... otherwise issue a read to the appropriate data endpoint,
+              //     which will return a read ticket - which can be used later
+              //     in the call promise to asynchronously wait for those bytes
+              //     to be available.
+              [this, frame_header]() -> absl::StatusOr<IncomingFrame> {
+                const auto padding =
+                    frame_header.Padding(options_.decode_alignment);
+                return IncomingFrame(
+                    frame_header,
+                    data_endpoints_.Read(frame_header.payload_connection_id - 1,
+                                         frame_header.payload_length + padding),
+                    padding);
+              });
         });
   }
 
@@ -135,10 +236,10 @@ class ChaoticGoodTransport : public RefCounted<ChaoticGoodTransport> {
   }
 
  private:
-  PromiseEndpoint control_endpoint_;
+  std::shared_ptr<grpc_event_engine::experimental::EventEngine> event_engine_;
-  PromiseEndpoint data_endpoint_;
+  ControlEndpoint control_endpoint_;
-  FrameHeader current_frame_header_;
+  DataEndpoints data_endpoints_;
-  Options options_;
+  const Options options_;
 };
 
 }  // namespace chaotic_good

src/core/ext/transport/chaotic_good/client/chaotic_good_connector.cc

@@ -46,6 +46,7 @@
 #include "src/core/lib/iomgr/event_engine_shims/endpoint.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/all_ok.h"
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/event_engine_wakeup_scheduler.h"
 #include "src/core/lib/promise/latch.h"
@@ -90,48 +91,53 @@ ChaoticGoodConnector::~ChaoticGoodConnector() {
 }
 
 auto ChaoticGoodConnector::DataEndpointReadSettingsFrame(
-    RefCountedPtr<ChaoticGoodConnector> self) {
+    RefCountedPtr<ChaoticGoodConnector> self, uint32_t data_connection_index) {
-  return TrySeq(self->data_endpoint_.ReadSlice(FrameHeader::kFrameHeaderSize),
+  return TrySeq(
-                [self](Slice slice) mutable {
+      self->data_endpoints_[data_connection_index].ReadSlice(
-                  // Read setting frame;
+          FrameHeader::kFrameHeaderSize),
-                  // Parse frame header
+      [self, data_connection_index](Slice slice) mutable {
-                  auto frame_header_ =
+        // Read setting frame;
-                      FrameHeader::Parse(reinterpret_cast<const uint8_t*>(
+        // Parse frame header
-                          GRPC_SLICE_START_PTR(slice.c_slice())));
+        auto frame_header_ =
-                  return If(
+            FrameHeader::Parse(reinterpret_cast<const uint8_t*>(
-                      frame_header_.ok(),
+                GRPC_SLICE_START_PTR(slice.c_slice())));
-                      [frame_header_ = *frame_header_, self]() {
+        return If(
-                        auto frame_header_length = frame_header_.payload_length;
+            frame_header_.ok(),
-                        return TrySeq(
+            [data_connection_index, frame_header_ = *frame_header_, self]() {
-                            self->data_endpoint_.Read(frame_header_length),
+              auto frame_header_length = frame_header_.payload_length;
+              return TrySeq(self->data_endpoints_[data_connection_index].Read(
+                                frame_header_length),
                             []() { return absl::OkStatus(); });
-                      },
+            },
-                      [status = frame_header_.status()]() { return status; });
+            [status = frame_header_.status()]() { return status; });
-                });
+      });
 }
 
 auto ChaoticGoodConnector::DataEndpointWriteSettingsFrame(
-    RefCountedPtr<ChaoticGoodConnector> self) {
+    RefCountedPtr<ChaoticGoodConnector> self, uint32_t data_connection_index) {
   // Serialize setting frame.
   SettingsFrame frame;
   frame.settings.set_data_channel(true);
-  frame.settings.set_connection_id(self->connection_id_);
+  frame.settings.add_connection_id(
+      self->connection_ids_[data_connection_index]);
   frame.settings.set_alignment(kDataAlignmentBytes);
   SliceBuffer write_buffer;
   frame.MakeHeader().Serialize(
       write_buffer.AddTiny(FrameHeader::kFrameHeaderSize));
   frame.SerializePayload(write_buffer);
   // ignore encoding errors: they will be logged separately already
-  return self->data_endpoint_.Write(std::move(write_buffer));
+  return self->data_endpoints_[data_connection_index].Write(
+      std::move(write_buffer));
 }
 
 auto ChaoticGoodConnector::WaitForDataEndpointSetup(
-    RefCountedPtr<ChaoticGoodConnector> self) {
+    RefCountedPtr<ChaoticGoodConnector> self, uint32_t data_connection_index) {
   // Data endpoint on_connect callback.
   grpc_event_engine::experimental::EventEngine::OnConnectCallback
       on_data_endpoint_connect =
-          [self](absl::StatusOr<std::unique_ptr<EventEngine::Endpoint>>
+          [self, data_connection_index](
-                     endpoint) mutable {
+              absl::StatusOr<std::unique_ptr<EventEngine::Endpoint>>
+                  endpoint) mutable {
             ExecCtx exec_ctx;
             if (!endpoint.ok() || self->handshake_mgr_ == nullptr) {
               MutexLock lock(&self->mu_);
@@ -150,9 +156,9 @@ auto ChaoticGoodConnector::WaitForDataEndpointSetup(
               chaotic_good_ext->EnableStatsCollection(
                   /*is_control_channel=*/false);
             }
-            self->data_endpoint_ =
+            self->data_endpoints_[data_connection_index] =
                 PromiseEndpoint(std::move(endpoint.value()), SliceBuffer());
-            self->data_endpoint_ready_.Set();
+            self->data_endpoint_ready_[data_connection_index]->Set();
           };
   self->event_engine_->Connect(
       std::move(on_data_endpoint_connect), *self->resolved_addr_,
@@ -161,13 +167,13 @@ auto ChaoticGoodConnector::WaitForDataEndpointSetup(
       ResourceQuota::Default()->memory_quota()->CreateMemoryAllocator(
           "data_endpoint_connection"),
       std::chrono::seconds(kTimeoutSecs));
-
   return TrySeq(Race(
-      TrySeq(self->data_endpoint_ready_.Wait(),
+      TrySeq(self->data_endpoint_ready_[data_connection_index]->Wait(),
-             [self]() mutable {
+             [self, data_connection_index]() mutable {
-               return TrySeq(DataEndpointWriteSettingsFrame(self),
+               return TrySeq(
-                             DataEndpointReadSettingsFrame(self),
+                   DataEndpointWriteSettingsFrame(self, data_connection_index),
-                             []() -> absl::Status { return absl::OkStatus(); });
+                   DataEndpointReadSettingsFrame(self, data_connection_index),
+                   []() -> absl::Status { return absl::OkStatus(); });
              }),
       TrySeq(Sleep(Timestamp::Now() + Duration::Seconds(kTimeoutSecs)),
              []() -> absl::Status {
@@ -198,10 +204,29 @@ auto ChaoticGoodConnector::ControlEndpointReadSettingsFrame(
                     return absl::UnavailableError(
                         "no connection id in settings frame");
                   }
-                  self->connection_id_ = frame.settings.connection_id();
+                  for (const auto& connection_id :
+                       frame.settings.connection_id()) {
+                    self->connection_ids_.push_back(connection_id);
+                  }
+                  self->data_endpoints_.resize(self->connection_ids_.size());
+                  for (size_t i = 0; i < self->connection_ids_.size(); ++i) {
+                    self->data_endpoint_ready_.emplace_back(
+                        std::make_unique<InterActivityLatch<void>>());
+                  }
                   return absl::OkStatus();
                 },
-                WaitForDataEndpointSetup(self)),
+                [self]() {
+                  // TODO(ctiller): find a better way than this
+                  std::vector<uint32_t> connection_ids;
+                  for (uint32_t i = 0; i < self->connection_ids_.size(); i++) {
+                    connection_ids.push_back(i);
+                  }
+                  return AllOkIter<absl::Status>(
+                      connection_ids.begin(), connection_ids.end(),
+                      [self](uint32_t connection_id) {
+                        return WaitForDataEndpointSetup(self, connection_id);
+                      });
+                }),
             [status = frame_header.status()]() { return status; });
       });
 }
@@ -318,7 +343,7 @@ void ChaoticGoodConnector::OnHandshakeDone(
           if (status.ok()) {
             self->result_->transport = new ChaoticGoodClientTransport(
                 std::move(self->control_endpoint_),
-                std::move(self->data_endpoint_), self->args_.channel_args,
+                std::move(self->data_endpoints_), self->args_.channel_args,
                 self->event_engine_);
             self->result_->channel_args = self->args_.channel_args;
             ExecCtx::Run(DEBUG_LOCATION, std::exchange(self->notify_, nullptr),

src/core/ext/transport/chaotic_good/client/chaotic_good_connector.h

@@ -65,15 +65,15 @@ class ChaoticGoodConnector : public SubchannelConnector {
 
  private:
   static auto DataEndpointReadSettingsFrame(
-      RefCountedPtr<ChaoticGoodConnector> self);
+      RefCountedPtr<ChaoticGoodConnector> self, uint32_t data_connection_index);
   static auto DataEndpointWriteSettingsFrame(
-      RefCountedPtr<ChaoticGoodConnector> self);
+      RefCountedPtr<ChaoticGoodConnector> self, uint32_t data_connection_index);
   static auto ControlEndpointReadSettingsFrame(
       RefCountedPtr<ChaoticGoodConnector> self);
   static auto ControlEndpointWriteSettingsFrame(
       RefCountedPtr<ChaoticGoodConnector> self);
-  static auto WaitForDataEndpointSetup(
+  static auto WaitForDataEndpointSetup(RefCountedPtr<ChaoticGoodConnector> self,
-      RefCountedPtr<ChaoticGoodConnector> self);
+                                       uint32_t data_connection_index);
   void OnHandshakeDone(absl::StatusOr<HandshakerArgs*> result);
 
   RefCountedPtr<Arena> arena_ = SimpleArenaAllocator()->MakeArena();
@@ -86,13 +86,13 @@ class ChaoticGoodConnector : public SubchannelConnector {
       resolved_addr_;
 
   PromiseEndpoint control_endpoint_;
-  PromiseEndpoint data_endpoint_;
+  std::vector<PromiseEndpoint> data_endpoints_;
+  std::vector<std::string> connection_ids_;
   ActivityPtr connect_activity_ ABSL_GUARDED_BY(mu_);
   const std::shared_ptr<grpc_event_engine::experimental::EventEngine>
       event_engine_;
   RefCountedPtr<HandshakeManager> handshake_mgr_;
-  InterActivityLatch<void> data_endpoint_ready_;
+  std::vector<std::unique_ptr<InterActivityLatch<void>>> data_endpoint_ready_;
-  std::string connection_id_;
 };
 }  // namespace chaotic_good
 }  // namespace grpc_core

src/core/ext/transport/chaotic_good/client_transport.cc

@@ -62,25 +62,6 @@ void ChaoticGoodClientTransport::Orphan() {
   Unref();
 }
 
-auto ChaoticGoodClientTransport::TransportWriteLoop(
-    RefCountedPtr<ChaoticGoodTransport> transport) {
-  return Loop([this, transport = std::move(transport)] {
-    return TrySeq(
-        // Get next outgoing frame.
-        outgoing_frames_.Next(),
-        // Serialize and write it out.
-        [transport = transport.get()](ClientFrame client_frame) {
-          return transport->WriteFrame(GetFrameInterface(client_frame));
-        },
-        []() -> LoopCtl<absl::Status> {
-          // The write failures will be caught in TrySeq and exit loop.
-          // Therefore, only need to return Continue() in the last lambda
-          // function.
-          return Continue();
-        });
-  });
-}
-
 absl::optional<CallHandler> ChaoticGoodClientTransport::LookupStream(
     uint32_t stream_id) {
   MutexLock lock(&mu_);
@@ -119,31 +100,39 @@ auto ChaoticGoodClientTransport::PushFrameIntoCall(
 }
 
 template <typename T>
-auto ChaoticGoodClientTransport::DispatchFrame(ChaoticGoodTransport* transport,
+auto ChaoticGoodClientTransport::DispatchFrame(
-                                               const FrameHeader& header,
+    RefCountedPtr<ChaoticGoodTransport> transport,
-                                               SliceBuffer payload) {
+    IncomingFrame incoming_frame) {
+  absl::optional<CallHandler> call_handler =
+      LookupStream(incoming_frame.header().stream_id);
   return GRPC_LATENT_SEE_PROMISE(
       "ChaoticGoodClientTransport::DispatchFrame",
-      TrySeq(
+      If(
-          [transport, header, payload = std::move(payload)]() mutable {
+          call_handler.has_value(),
-            return transport->DeserializeFrame<T>(header, std::move(payload));
+          [this, &call_handler, &incoming_frame, &transport]() {
-          },
+            // TODO(ctiller): instead of SpawnWaitable here we probably want a
-          [this](T frame) {
+            // small queue to push into, so that the call can proceed
-            absl::optional<CallHandler> call_handler =
+            // asynchronously to other calls regardless of frame ordering.
-                LookupStream(frame.stream_id);
+            return call_handler->SpawnWaitable(
-            return If(
+                "push-frame", [this, call_handler = *call_handler,
-                call_handler.has_value(),
+                               incoming_frame = std::move(incoming_frame),
-                [this, &call_handler, &frame]() {
+                               transport = std::move(transport)]() mutable {
-                  return call_handler->SpawnWaitable(
+                  return TrySeq(
-                      "push-frame", [this, call_handler = *call_handler,
+                      incoming_frame.Payload(),
-                                     frame = std::move(frame)]() mutable {
+                      [transport = std::move(transport),
+                       header = incoming_frame.header()](SliceBuffer payload) {
+                        return transport->DeserializeFrame<T>(
+                            header, std::move(payload));
+                      },
+                      [call_handler, this](T frame) mutable {
                         return Map(call_handler.CancelIfFails(PushFrameIntoCall(
                                        std::move(frame), call_handler)),
                                    [](StatusFlag) { return absl::OkStatus(); });
-                      });
+                      },
-                },
+                      ImmediateOkStatus());
-                []() { return absl::OkStatus(); });
+                });
-          }));
+          },
+          []() { return absl::OkStatus(); }));
 }
 
 auto ChaoticGoodClientTransport::TransportReadLoop(
@@ -151,27 +140,24 @@ auto ChaoticGoodClientTransport::TransportReadLoop(
   return Loop([this, transport = std::move(transport)] {
     return TrySeq(
         transport->ReadFrameBytes(),
-        [this, transport = transport.get()](
+        [this, transport](IncomingFrame incoming_frame) {
-            std::tuple<FrameHeader, SliceBuffer> frame_bytes) {
-          const auto& header = std::get<0>(frame_bytes);
-          SliceBuffer& payload = std::get<1>(frame_bytes);
           return Switch(
-              header.type,
+              incoming_frame.header().type,
               Case<FrameType, FrameType::kServerInitialMetadata>([&, this]() {
                 return DispatchFrame<ServerInitialMetadataFrame>(
-                    transport, header, std::move(payload));
+                    transport, std::move(incoming_frame));
               }),
               Case<FrameType, FrameType::kServerTrailingMetadata>([&, this]() {
                 return DispatchFrame<ServerTrailingMetadataFrame>(
-                    transport, header, std::move(payload));
+                    transport, std::move(incoming_frame));
               }),
               Case<FrameType, FrameType::kMessage>([&, this]() {
-                return DispatchFrame<MessageFrame>(transport, header,
+                return DispatchFrame<MessageFrame>(transport,
-                                                   std::move(payload));
+                                                   std::move(incoming_frame));
               }),
               Default([&]() {
                 LOG_EVERY_N_SEC(INFO, 10)
-                    << "Bad frame type: " << header.ToString();
+                    << "Bad frame type: " << incoming_frame.header().ToString();
                 return absl::OkStatus();
               }));
         },
@@ -191,36 +177,42 @@ auto ChaoticGoodClientTransport::OnTransportActivityDone(
 }
 
 ChaoticGoodClientTransport::ChaoticGoodClientTransport(
-    PromiseEndpoint control_endpoint, PromiseEndpoint data_endpoint,
+    PromiseEndpoint control_endpoint,
-    const ChannelArgs& args,
+    std::vector<PromiseEndpoint> data_endpoints, const ChannelArgs& args,
     std::shared_ptr<grpc_event_engine::experimental::EventEngine> event_engine)
     : allocator_(args.GetObject<ResourceQuota>()
                      ->memory_quota()
                      ->CreateMemoryAllocator("chaotic-good")),
       outgoing_frames_(4) {
+  CHECK(event_engine != nullptr);
   // Set up TCP tracer if enabled.
   if (args.GetBool(GRPC_ARG_TCP_TRACING_ENABLED).value_or(false)) {
-    auto* epte = grpc_event_engine::experimental::QueryExtension<
+    for (auto& ep : data_endpoints) {
-        grpc_event_engine::experimental::TcpTraceExtension>(
+      auto* epte = grpc_event_engine::experimental::QueryExtension<
-        data_endpoint.GetEventEngineEndpoint().get());
+          grpc_event_engine::experimental::TcpTraceExtension>(
-    if (epte != nullptr) {
+          ep.GetEventEngineEndpoint().get());
-      epte->InitializeAndReturnTcpTracer();
+      if (epte != nullptr) {
+        epte->InitializeAndReturnTcpTracer();
+      }
     }
   }
+  CHECK(event_engine != nullptr);
   ChaoticGoodTransport::Options options;
   options.inlined_payload_size_threshold =
       args.GetInt("grpc.chaotic_good.inlined_payload_size_threshold")
           .value_or(options.inlined_payload_size_threshold);
   auto transport = MakeRefCounted<ChaoticGoodTransport>(
-      std::move(control_endpoint), std::move(data_endpoint), options);
+      std::move(control_endpoint), std::move(data_endpoints), event_engine,
+      options);
   auto party_arena = SimpleArenaAllocator(0)->MakeArena();
   party_arena->SetContext<grpc_event_engine::experimental::EventEngine>(
       event_engine.get());
   party_ = Party::Make(std::move(party_arena));
-  party_->Spawn("client-chaotic-writer",
+  party_->Spawn(
-                GRPC_LATENT_SEE_PROMISE("ClientTransportWriteLoop",
+      "client-chaotic-writer",
-                                        TransportWriteLoop(transport)),
+      GRPC_LATENT_SEE_PROMISE("ClientTransportWriteLoop",
-                OnTransportActivityDone("write_loop"));
+                              transport->TransportWriteLoop(outgoing_frames_)),
+      OnTransportActivityDone("write_loop"));
   party_->Spawn(
       "client-chaotic-reader",
       GRPC_LATENT_SEE_PROMISE("ClientTransportReadLoop",

src/core/ext/transport/chaotic_good/client_transport.h

@@ -64,7 +64,8 @@ namespace chaotic_good {
 class ChaoticGoodClientTransport final : public ClientTransport {
  public:
   ChaoticGoodClientTransport(
-      PromiseEndpoint control_endpoint, PromiseEndpoint data_endpoint,
+      PromiseEndpoint control_endpoint,
+      std::vector<PromiseEndpoint> data_endpoints,
       const ChannelArgs& channel_args,
       std::shared_ptr<grpc_event_engine::experimental::EventEngine>
           event_engine);
@@ -89,10 +90,9 @@ class ChaoticGoodClientTransport final : public ClientTransport {
   absl::optional<CallHandler> LookupStream(uint32_t stream_id);
   auto CallOutboundLoop(uint32_t stream_id, CallHandler call_handler);
   auto OnTransportActivityDone(absl::string_view what);
-  auto TransportWriteLoop(RefCountedPtr<ChaoticGoodTransport> transport);
   template <typename T>
-  auto DispatchFrame(ChaoticGoodTransport* transport, const FrameHeader& header,
+  auto DispatchFrame(RefCountedPtr<ChaoticGoodTransport> transport,
-                     SliceBuffer payload);
+                     IncomingFrame incoming_frame);
   auto TransportReadLoop(RefCountedPtr<ChaoticGoodTransport> transport);
   // Push one frame into a call
   auto PushFrameIntoCall(ServerInitialMetadataFrame frame,

src/core/ext/transport/chaotic_good/control_endpoint.cc

@@ -0,0 +1,68 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/core/ext/transport/chaotic_good/control_endpoint.h"
+
+#include "src/core/lib/event_engine/event_engine_context.h"
+#include "src/core/lib/event_engine/tcp_socket_utils.h"
+#include "src/core/lib/promise/loop.h"
+#include "src/core/lib/promise/try_seq.h"
+
+namespace grpc_core {
+namespace chaotic_good {
+
+auto ControlEndpoint::Buffer::Pull() {
+  return [this]() -> Poll<SliceBuffer> {
+    Waker waker;
+    auto cleanup = absl::MakeCleanup([&waker]() { waker.Wakeup(); });
+    MutexLock lock(&mu_);
+    if (queued_output_.Length() == 0) {
+      flush_waker_ = GetContext<Activity>()->MakeNonOwningWaker();
+      return Pending{};
+    }
+    waker = std::move(write_waker_);
+    return std::move(queued_output_);
+  };
+}
+
+ControlEndpoint::ControlEndpoint(
+    PromiseEndpoint endpoint,
+    grpc_event_engine::experimental::EventEngine* event_engine)
+    : endpoint_(std::make_shared<PromiseEndpoint>(std::move(endpoint))) {
+  CHECK(event_engine != nullptr);
+  write_party_->arena()->SetContext(event_engine);
+  write_party_->Spawn(
+      "flush-control",
+      GRPC_LATENT_SEE_PROMISE(
+          "FlushLoop", Loop([endpoint = endpoint_, buffer = buffer_]() {
+            return TrySeq(
+                // Pull one set of buffered writes
+                buffer->Pull(),
+                // And write them
+                [endpoint, buffer = buffer.get()](SliceBuffer flushing) {
+                  GRPC_TRACE_LOG(chaotic_good, INFO)
+                      << "CHAOTIC_GOOD: Flush " << flushing.Length()
+                      << " bytes from " << buffer << " to "
+                      << ResolvedAddressToString(endpoint->GetPeerAddress())
+                             .value_or("<<unknown peer address>>");
+                  return endpoint->Write(std::move(flushing));
+                },
+                // Then repeat
+                []() -> LoopCtl<absl::Status> { return Continue{}; });
+          })),
+      [](absl::Status) {});
+}
+
+}  // namespace chaotic_good
+}  // namespace grpc_core

src/core/ext/transport/chaotic_good/control_endpoint.h

@@ -0,0 +1,99 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_EXT_TRANSPORT_CHAOTIC_GOOD_CONTROL_ENDPOINT_H
+#define GRPC_SRC_CORE_EXT_TRANSPORT_CHAOTIC_GOOD_CONTROL_ENDPOINT_H
+
+#include "absl/cleanup/cleanup.h"
+#include "src/core/lib/promise/party.h"
+#include "src/core/lib/transport/promise_endpoint.h"
+#include "src/core/util/sync.h"
+
+namespace grpc_core {
+namespace chaotic_good {
+
+// Wrapper around PromiseEndpoint.
+// Buffers all of the small writes that get enqueued to this endpoint, and then
+// uses a separate party to flush them to the wire.
+// In doing so we get to batch up effectively all the writes from the transport
+// (since party wakeups are sticky), and then flush all the writes in one go.
+class ControlEndpoint {
+ private:
+  class Buffer : public RefCounted<Buffer> {
+   public:
+    // Queue some buffer to be written.
+    // We cap the queue size so that we don't infinitely buffer on one
+    // connection - if the cap is hit, this queue operation will not resolve
+    // until it empties.
+    // Returns a promise that resolves to Empty{} when the data has been queued.
+    auto Queue(SliceBuffer&& buffer) {
+      return [buffer = std::move(buffer), this]() mutable -> Poll<Empty> {
+        Waker waker;
+        auto cleanup = absl::MakeCleanup([&waker]() { waker.Wakeup(); });
+        MutexLock lock(&mu_);
+        if (queued_output_.Length() != 0 &&
+            queued_output_.Length() + buffer.Length() > MaxQueued()) {
+          GRPC_TRACE_LOG(chaotic_good, INFO)
+              << "CHAOTIC_GOOD: Delay control write"
+              << " write_length=" << buffer.Length()
+              << " already_buffered=" << queued_output_.Length()
+              << " queue=" << this;
+          write_waker_ = GetContext<Activity>()->MakeNonOwningWaker();
+          return Pending{};
+        }
+        GRPC_TRACE_LOG(chaotic_good, INFO)
+            << "CHAOTIC_GOOD: Queue control write " << buffer.Length()
+            << " bytes on " << this;
+        waker = std::move(flush_waker_);
+        queued_output_.Append(buffer);
+        return Empty{};
+      };
+    }
+
+    auto Pull();
+
+   private:
+    size_t MaxQueued() const { return 1024 * 1024; }
+
+    Mutex mu_;
+    Waker write_waker_ ABSL_GUARDED_BY(mu_);
+    Waker flush_waker_ ABSL_GUARDED_BY(mu_);
+    SliceBuffer queued_output_ ABSL_GUARDED_BY(mu_);
+  };
+
+ public:
+  ControlEndpoint(PromiseEndpoint endpoint,
+                  grpc_event_engine::experimental::EventEngine* event_engine);
+
+  // Write some data to the control endpoint; returns a promise that resolves
+  // to Empty{} -- it's not possible to see errors from this api.
+  auto Write(SliceBuffer&& bytes) { return buffer_->Queue(std::move(bytes)); }
+
+  // Read operations are simply passthroughs to the underlying promise endpoint.
+  auto ReadSlice(size_t length) { return endpoint_->ReadSlice(length); }
+  auto Read(size_t length) { return endpoint_->Read(length); }
+  auto GetPeerAddress() const { return endpoint_->GetPeerAddress(); }
+  auto GetLocalAddress() const { return endpoint_->GetLocalAddress(); }
+
+ private:
+  std::shared_ptr<PromiseEndpoint> endpoint_;
+  RefCountedPtr<Party> write_party_ =
+      Party::Make(SimpleArenaAllocator(0)->MakeArena());
+  RefCountedPtr<Buffer> buffer_ = MakeRefCounted<Buffer>();
+};
+
+}  // namespace chaotic_good
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_EXT_TRANSPORT_CHAOTIC_GOOD_CONTROL_ENDPOINT_H

src/core/ext/transport/chaotic_good/data_endpoints.cc

@@ -0,0 +1,236 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/core/ext/transport/chaotic_good/data_endpoints.h"
+
+#include <cstddef>
+
+#include "absl/cleanup/cleanup.h"
+#include "absl/strings/escaping.h"
+#include "src/core/lib/event_engine/event_engine_context.h"
+#include "src/core/lib/promise/loop.h"
+#include "src/core/lib/promise/seq.h"
+#include "src/core/lib/promise/try_seq.h"
+
+namespace grpc_core {
+namespace chaotic_good {
+
+namespace data_endpoints_detail {
+
+///////////////////////////////////////////////////////////////////////////////
+// OutputBuffer
+
+bool OutputBuffer::Accept(SliceBuffer& buffer) {
+  if (pending_.Length() != 0 &&
+      pending_.Length() + buffer.Length() > pending_max_) {
+    return false;
+  }
+  pending_.Append(buffer);
+  return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// OutputBuffers
+
+OutputBuffers::OutputBuffers(uint32_t num_connections)
+    : buffers_(num_connections) {}
+
+Poll<uint32_t> OutputBuffers::PollWrite(SliceBuffer& output_buffer) {
+  Waker waker;
+  auto cleanup = absl::MakeCleanup([&waker]() { waker.Wakeup(); });
+  const auto length = output_buffer.Length();
+  MutexLock lock(&mu_);
+  for (size_t i = 0; i < buffers_.size(); ++i) {
+    if (buffers_[i].Accept(output_buffer)) {
+      GRPC_TRACE_LOG(chaotic_good, INFO)
+          << "CHAOTIC_GOOD: Queue " << length << " data onto endpoint " << i
+          << " queue " << this;
+      waker = buffers_[i].TakeWaker();
+      return i;
+    }
+  }
+  GRPC_TRACE_LOG(chaotic_good, INFO)
+      << "CHAOTIC_GOOD: No data endpoint ready for " << length
+      << " bytes on queue " << this;
+  write_waker_ = GetContext<Activity>()->MakeNonOwningWaker();
+  return Pending{};
+}
+
+Poll<SliceBuffer> OutputBuffers::PollNext(uint32_t connection_id) {
+  Waker waker;
+  auto cleanup = absl::MakeCleanup([&waker]() { waker.Wakeup(); });
+  MutexLock lock(&mu_);
+  auto& buffer = buffers_[connection_id];
+  if (buffer.HavePending()) {
+    waker = std::move(write_waker_);
+    return buffer.TakePending();
+  }
+  buffer.SetWaker();
+  return Pending{};
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// InputQueues
+
+InputQueues::InputQueues(uint32_t num_connections)
+    : read_requests_(num_connections), read_request_waker_(num_connections) {}
+
+absl::StatusOr<uint64_t> InputQueues::CreateTicket(uint32_t connection_id,
+                                                   size_t length) {
+  Waker waker;
+  auto cleanup = absl::MakeCleanup([&waker]() { waker.Wakeup(); });
+  MutexLock lock(&mu_);
+  if (connection_id >= read_requests_.size()) {
+    return absl::UnavailableError(
+        absl::StrCat("Invalid connection id: ", connection_id));
+  }
+  uint64_t ticket = next_ticket_id_;
+  ++next_ticket_id_;
+  auto r = ReadRequest{length, ticket};
+  GRPC_TRACE_LOG(chaotic_good, INFO)
+      << "CHAOTIC_GOOD: New read ticket on #" << connection_id << " " << r;
+  read_requests_[connection_id].push_back(r);
+  outstanding_reads_.emplace(ticket, Waker{});
+  waker = std::move(read_request_waker_[connection_id]);
+  return ticket;
+}
+
+Poll<absl::StatusOr<SliceBuffer>> InputQueues::PollRead(uint64_t ticket) {
+  MutexLock lock(&mu_);
+  auto it = outstanding_reads_.find(ticket);
+  CHECK(it != outstanding_reads_.end()) << " ticket=" << ticket;
+  if (auto* waker = absl::get_if<Waker>(&it->second)) {
+    *waker = GetContext<Activity>()->MakeNonOwningWaker();
+    return Pending{};
+  }
+  auto result = std::move(absl::get<absl::StatusOr<SliceBuffer>>(it->second));
+  outstanding_reads_.erase(it);
+  GRPC_TRACE_LOG(chaotic_good, INFO)
+      << "CHAOTIC_GOOD: Poll for ticket #" << ticket
+      << " completes: " << result.status();
+  return result;
+}
+
+Poll<std::vector<InputQueues::ReadRequest>> InputQueues::PollNext(
+    uint32_t connection_id) {
+  MutexLock lock(&mu_);
+  auto& q = read_requests_[connection_id];
+  if (q.empty()) {
+    read_request_waker_[connection_id] =
+        GetContext<Activity>()->MakeNonOwningWaker();
+    return Pending{};
+  }
+  auto r = std::move(q);
+  q.clear();
+  return r;
+}
+
+void InputQueues::CompleteRead(uint64_t ticket,
+                               absl::StatusOr<SliceBuffer> buffer) {
+  Waker waker;
+  auto cleanup = absl::MakeCleanup([&waker]() { waker.Wakeup(); });
+  MutexLock lock(&mu_);
+  GRPC_TRACE_LOG(chaotic_good, INFO)
+      << "CHAOTIC_GOOD: Complete ticket #" << ticket << ": " << buffer.status();
+  auto it = outstanding_reads_.find(ticket);
+  if (it == outstanding_reads_.end()) return;  // cancelled
+  waker = std::move(absl::get<Waker>(it->second));
+  it->second.emplace<absl::StatusOr<SliceBuffer>>(std::move(buffer));
+}
+
+void InputQueues::CancelTicket(uint64_t ticket) {
+  MutexLock lock(&mu_);
+  outstanding_reads_.erase(ticket);
+}
+
+}  // namespace data_endpoints_detail
+
+///////////////////////////////////////////////////////////////////////////////
+// DataEndpoints
+
+DataEndpoints::DataEndpoints(
+    std::vector<PromiseEndpoint> endpoints_vec,
+    grpc_event_engine::experimental::EventEngine* event_engine)
+    : output_buffers_(MakeRefCounted<data_endpoints_detail::OutputBuffers>(
+          endpoints_vec.size())),
+      input_queues_(MakeRefCounted<data_endpoints_detail::InputQueues>(
+          endpoints_vec.size())) {
+  CHECK(event_engine != nullptr);
+  for (auto& endpoint : endpoints_vec) {
+    // Enable RxMemoryAlignment and RPC receive coalescing after the transport
+    // setup is complete. At this point all the settings frames should have
+    // been read.
+    endpoint.EnforceRxMemoryAlignmentAndCoalescing();
+  }
+  auto endpoints = MakeRefCounted<data_endpoints_detail::Endpoints>();
+  endpoints->endpoints = std::move(endpoints_vec);
+  parties_.reserve(2 * endpoints->endpoints.size());
+  auto arena = SimpleArenaAllocator(0)->MakeArena();
+  arena->SetContext(event_engine);
+  for (size_t i = 0; i < endpoints->endpoints.size(); ++i) {
+    auto write_party = Party::Make(arena);
+    auto read_party = Party::Make(arena);
+    write_party->Spawn(
+        "flush-data",
+        [i, endpoints, output_buffers = output_buffers_]() {
+          return Loop([i, endpoints, output_buffers]() {
+            return TrySeq(
+                output_buffers->Next(i),
+                [endpoints = endpoints.get(), i](SliceBuffer buffer) {
+                  GRPC_TRACE_LOG(chaotic_good, INFO)
+                      << "CHAOTIC_GOOD: Write " << buffer.Length()
+                      << "b to data endpoint #" << i;
+                  return endpoints->endpoints[i].Write(std::move(buffer));
+                },
+                []() -> LoopCtl<absl::Status> { return Continue{}; });
+          });
+        },
+        [](absl::Status) {});
+    read_party->Spawn(
+        "read-data",
+        [i, endpoints, input_queues = input_queues_]() {
+          return Loop([i, endpoints, input_queues]() {
+            return TrySeq(
+                input_queues->Next(i),
+                [endpoints, i, input_queues](
+                    std::vector<data_endpoints_detail::InputQueues::ReadRequest>
+                        requests) {
+                  return TrySeqContainer(
+                      std::move(requests), Empty{},
+                      [endpoints, i, input_queues](
+                          data_endpoints_detail::InputQueues::ReadRequest
+                              read_request,
+                          Empty) {
+                        return Seq(
+                            endpoints->endpoints[i].Read(read_request.length),
+                            [ticket = read_request.ticket,
+
+                             input_queues](absl::StatusOr<SliceBuffer> buffer) {
+                              input_queues->CompleteRead(ticket,
+                                                         std::move(buffer));
+                              return Empty{};
+                            });
+                      });
+                },
+                []() -> LoopCtl<absl::Status> { return Continue{}; });
+          });
+        },
+        [](absl::Status) {});
+    parties_.emplace_back(std::move(write_party));
+    parties_.emplace_back(std::move(read_party));
+  }
+}
+
+}  // namespace chaotic_good
+}  // namespace grpc_core

src/core/ext/transport/chaotic_good/data_endpoints.h

@@ -0,0 +1,199 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_EXT_TRANSPORT_CHAOTIC_GOOD_DATA_ENDPOINTS_H
+#define GRPC_SRC_CORE_EXT_TRANSPORT_CHAOTIC_GOOD_DATA_ENDPOINTS_H
+
+#include <cstdint>
+
+#include "src/core/lib/promise/party.h"
+#include "src/core/lib/promise/promise.h"
+#include "src/core/lib/slice/slice_buffer.h"
+#include "src/core/lib/transport/promise_endpoint.h"
+
+namespace grpc_core {
+namespace chaotic_good {
+
+namespace data_endpoints_detail {
+struct Endpoints : public RefCounted<Endpoints> {
+  std::vector<PromiseEndpoint> endpoints;
+};
+
+// Buffered writes for one data endpoint
+class OutputBuffer {
+ public:
+  bool Accept(SliceBuffer& buffer);
+  Waker TakeWaker() { return std::move(flush_waker_); }
+  void SetWaker() {
+    flush_waker_ = GetContext<Activity>()->MakeNonOwningWaker();
+  }
+  bool HavePending() const { return pending_.Length() > 0; }
+  SliceBuffer TakePending() { return std::move(pending_); }
+
+ private:
+  Waker flush_waker_;
+  size_t pending_max_ = 1024 * 1024;
+  SliceBuffer pending_;
+};
+
+// The set of output buffers for all connected data endpoints
+class OutputBuffers : public RefCounted<OutputBuffers> {
+ public:
+  explicit OutputBuffers(uint32_t num_connections);
+
+  auto Write(SliceBuffer output_buffer) {
+    return [output_buffer = std::move(output_buffer), this]() mutable {
+      return PollWrite(output_buffer);
+    };
+  }
+
+  auto Next(uint32_t connection_id) {
+    return [this, connection_id]() { return PollNext(connection_id); };
+  }
+
+ private:
+  Poll<uint32_t> PollWrite(SliceBuffer& output_buffer);
+  Poll<SliceBuffer> PollNext(uint32_t connection_id);
+
+  Mutex mu_;
+  std::vector<OutputBuffer> buffers_ ABSL_GUARDED_BY(mu_);
+  Waker write_waker_ ABSL_GUARDED_BY(mu_);
+};
+
+class InputQueues : public RefCounted<InputQueues> {
+ public:
+  // One outstanding read.
+  // ReadTickets get filed by read requests, and all tickets are fullfilled
+  // by an endpoint.
+  // A call may Await a ticket to get the bytes back later (or it may skip that
+  // step - in which case the bytes are thrown away after reading).
+  // This decoupling is necessary to ensure that cancelled reads by calls do not
+  // cause data corruption for other calls.
+  class ReadTicket {
+   public:
+    ReadTicket(absl::StatusOr<uint64_t> ticket,
+               RefCountedPtr<InputQueues> input_queues)
+        : ticket_(std::move(ticket)), input_queues_(std::move(input_queues)) {}
+
+    ReadTicket(const ReadTicket&) = delete;
+    ReadTicket& operator=(const ReadTicket&) = delete;
+    ReadTicket(ReadTicket&& other) noexcept
+        : ticket_(std::move(other.ticket_)),
+          input_queues_(std::move(other.input_queues_)) {}
+    ReadTicket& operator=(ReadTicket&& other) noexcept {
+      ticket_ = std::move(other.ticket_);
+      input_queues_ = std::move(other.input_queues_);
+      return *this;
+    }
+
+    ~ReadTicket() {
+      if (input_queues_ != nullptr && ticket_.ok()) {
+        input_queues_->CancelTicket(*ticket_);
+      }
+    }
+
+    auto Await() {
+      return If(
+          ticket_.ok(),
+          [&]() {
+            return
+                [ticket = *ticket_, input_queues = std::move(input_queues_)]() {
+                  return input_queues->PollRead(ticket);
+                };
+          },
+          [&]() {
+            return Immediate(absl::StatusOr<SliceBuffer>(ticket_.status()));
+          });
+    }
+
+   private:
+    absl::StatusOr<uint64_t> ticket_;
+    RefCountedPtr<InputQueues> input_queues_;
+  };
+
+  struct ReadRequest {
+    size_t length;
+    uint64_t ticket;
+
+    template <typename Sink>
+    friend void AbslStringify(Sink& sink, const ReadRequest& req) {
+      sink.Append(absl::StrCat("read#", req.ticket, ":", req.length, "b"));
+    }
+  };
+
+  explicit InputQueues(uint32_t num_connections);
+
+  ReadTicket Read(uint32_t connection_id, size_t length) {
+    return ReadTicket(CreateTicket(connection_id, length), Ref());
+  }
+
+  auto Next(uint32_t connection_id) {
+    return [this, connection_id]() { return PollNext(connection_id); };
+  }
+
+  void CompleteRead(uint64_t ticket, absl::StatusOr<SliceBuffer> buffer);
+
+  void CancelTicket(uint64_t ticket);
+
+ private:
+  using ReadState = absl::variant<absl::StatusOr<SliceBuffer>, Waker>;
+
+  absl::StatusOr<uint64_t> CreateTicket(uint32_t connection_id, size_t length);
+  Poll<absl::StatusOr<SliceBuffer>> PollRead(uint64_t ticket);
+  Poll<std::vector<ReadRequest>> PollNext(uint32_t connection_id);
+
+  Mutex mu_;
+  uint64_t next_ticket_id_ ABSL_GUARDED_BY(mu_) = 0;
+  std::vector<std::vector<ReadRequest>> read_requests_ ABSL_GUARDED_BY(mu_);
+  std::vector<Waker> read_request_waker_;
+  absl::flat_hash_map<uint64_t, ReadState> outstanding_reads_
+      ABSL_GUARDED_BY(mu_);
+};
+}  // namespace data_endpoints_detail
+
+// Collection of data connections.
+class DataEndpoints {
+ public:
+  using ReadTicket = data_endpoints_detail::InputQueues::ReadTicket;
+
+  explicit DataEndpoints(
+      std::vector<PromiseEndpoint> endpoints,
+      grpc_event_engine::experimental::EventEngine* event_engine);
+
+  // Try to queue output_buffer against a data endpoint.
+  // Returns a promise that resolves to the data endpoint connection id
+  // selected.
+  // Connection ids returned by this class are 0 based (which is different
+  // to how chaotic good communicates them on the wire - those are 1 based
+  // to allow for the control channel identification)
+  auto Write(SliceBuffer output_buffer) {
+    return output_buffers_->Write(std::move(output_buffer));
+  }
+
+  ReadTicket Read(uint32_t connection_id, uint32_t length) {
+    return input_queues_->Read(connection_id, length);
+  }
+
+  bool empty() const { return parties_.empty(); }
+
+ private:
+  RefCountedPtr<data_endpoints_detail::OutputBuffers> output_buffers_;
+  RefCountedPtr<data_endpoints_detail::InputQueues> input_queues_;
+  std::vector<RefCountedPtr<Party>> parties_;
+};
+
+}  // namespace chaotic_good
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_EXT_TRANSPORT_CHAOTIC_GOOD_DATA_ENDPOINTS_H

src/core/ext/transport/chaotic_good/server/chaotic_good_server.cc

@@ -50,6 +50,7 @@
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/event_engine_wakeup_scheduler.h"
 #include "src/core/lib/promise/if.h"
+#include "src/core/lib/promise/join.h"
 #include "src/core/lib/promise/latch.h"
 #include "src/core/lib/promise/race.h"
 #include "src/core/lib/promise/sleep.h"
@@ -175,17 +176,21 @@ void ChaoticGoodServerListener::ActiveConnection::Orphan() {
   Unref();
 }
 
-void ChaoticGoodServerListener::ActiveConnection::NewConnectionID() {
+void ChaoticGoodServerListener::ActiveConnection::NewConnectionIDs(
-  bool has_new_id = false;
+    size_t count) {
   MutexLock lock(&listener_->mu_);
-  while (!has_new_id) {
+  for (size_t i = 0; i < count; i++) {
-    connection_id_ = listener_->connection_id_generator_();
+    std::string connection_id;
-    if (!listener_->connectivity_map_.contains(connection_id_)) {
+    while (true) {
-      has_new_id = true;
+      connection_id = listener_->connection_id_generator_();
+      if (!listener_->connectivity_map_.contains(connection_id)) {
+        break;
+      }
     }
+    listener_->connectivity_map_.emplace(
+        connection_id, std::make_shared<InterActivityLatch<PromiseEndpoint>>());
+    connection_ids_.emplace_back(std::move(connection_id));
   }
-  listener_->connectivity_map_.emplace(
-      connection_id_, std::make_shared<InterActivityLatch<PromiseEndpoint>>());
 }
 
 void ChaoticGoodServerListener::ActiveConnection::Done() {
@@ -246,13 +251,20 @@ auto ChaoticGoodServerListener::ActiveConnection::HandshakingState::
                         return absl::UnavailableError(
                             "no connection id in data endpoint settings frame");
                       }
+                      if (frame.settings.connection_id().size() != 1) {
+                        return absl::UnavailableError(absl::StrCat(
+                            "Got ", frame.settings.connection_id().size(),
+                            " connection ids in data endpoint "
+                            "settings frame (expect one)"));
+                      }
                       if (frame.settings.alignment() == 0) {
                         return absl::UnavailableError(
                             "no alignment in data endpoint settings frame");
                       }
                       // Get connection-id and data-alignment for data endpoint.
-                      self->connection_->connection_id_ =
+                      self->connection_->connection_ids_.clear();
-                          frame.settings.connection_id();
+                      self->connection_->connection_ids_.push_back(
+                          frame.settings.connection_id()[0]);
                       self->connection_->data_alignment_ =
                           frame.settings.alignment();
                     }
@@ -278,12 +290,18 @@ auto ChaoticGoodServerListener::ActiveConnection::HandshakingState::
           },
           [self]() {
             MutexLock lock(&self->connection_->listener_->mu_);
-            auto latch = self->connection_->listener_->connectivity_map_
+            return JoinIter(
-                             .find(self->connection_->connection_id_)
+                self->connection_->connection_ids_.begin(),
-                             ->second;
+                self->connection_->connection_ids_.end(),
-            return latch->Wait();
+                [self](const std::string& connection_id) {
+                  self->connection_->listener_->mu_.AssertHeld();
+                  auto latch = self->connection_->listener_->connectivity_map_
+                                   .find(connection_id)
+                                   ->second;
+                  return latch->Wait();
+                });
           },
-          [self](PromiseEndpoint ret) -> absl::Status {
+          [self](std::vector<PromiseEndpoint> ret) -> absl::Status {
             MutexLock lock(&self->connection_->listener_->mu_);
             GRPC_TRACE_LOG(chaotic_good, INFO)
                 << self->connection_.get()
@@ -305,19 +323,27 @@ auto ChaoticGoodServerListener::ActiveConnection::HandshakingState::
           Sleep(Timestamp::Now() + kConnectionDeadline),
           [self]() mutable -> absl::Status {
             MutexLock lock(&self->connection_->listener_->mu_);
-            // Delete connection id from map when timeout;
+            // Delete connection ids from map when timeout;
-            self->connection_->listener_->connectivity_map_.erase(
+            for (const std::string& connection_id :
-                self->connection_->connection_id_);
+                 self->connection_->connection_ids_) {
+              self->connection_->listener_->connectivity_map_.erase(
+                  connection_id);
+            }
             return absl::DeadlineExceededError("Deadline exceeded.");
           }));
 }
 
 auto ChaoticGoodServerListener::ActiveConnection::HandshakingState::
     ControlEndpointWriteSettingsFrame(RefCountedPtr<HandshakingState> self) {
-  self->connection_->NewConnectionID();
+  self->connection_->NewConnectionIDs(
+      self->connection_->listener_->args()
+          .GetInt(GRPC_ARG_CHAOTIC_GOOD_DATA_CONNECTIONS)
+          .value_or(1));
   SettingsFrame frame;
   frame.settings.set_data_channel(false);
-  frame.settings.set_connection_id(self->connection_->connection_id_);
+  for (const auto& connection_id : self->connection_->connection_ids_) {
+    frame.settings.add_connection_id(connection_id);
+  }
   SliceBuffer write_buffer;
   frame.MakeHeader().Serialize(
       write_buffer.AddTiny(FrameHeader::kFrameHeaderSize));
@@ -332,7 +358,6 @@ auto ChaoticGoodServerListener::ActiveConnection::HandshakingState::
   // Send data endpoint setting frame
   SettingsFrame frame;
   frame.settings.set_data_channel(true);
-  frame.settings.set_connection_id(self->connection_->connection_id_);
   frame.settings.set_alignment(self->connection_->data_alignment_);
   SliceBuffer write_buffer;
   frame.MakeHeader().Serialize(
@@ -344,12 +369,13 @@ auto ChaoticGoodServerListener::ActiveConnection::HandshakingState::
       [self]() mutable {
         MutexLock lock(&self->connection_->listener_->mu_);
         // Set endpoint to latch
+        CHECK_EQ(self->connection_->connection_ids_.size(), 1ull);
         auto it = self->connection_->listener_->connectivity_map_.find(
-            self->connection_->connection_id_);
+            self->connection_->connection_ids_[0]);
         if (it == self->connection_->listener_->connectivity_map_.end()) {
-          return absl::InternalError(
+          return absl::InternalError(absl::StrCat(
-              absl::StrCat("Connection not in map: ",
+              "Connection not in map: ",
-                           absl::CEscape(self->connection_->connection_id_)));
+              absl::CEscape(self->connection_->connection_ids_[0])));
         }
         it->second->Set(std::move(self->connection_->endpoint_));
         return absl::OkStatus();

src/core/ext/transport/chaotic_good/server/chaotic_good_server.h

@@ -44,6 +44,11 @@
 #include "src/core/util/sync.h"
 #include "src/core/util/time.h"
 
+// Channel arg: integer number of data connections to specify
+// Defaults to 1 if not set
+#define GRPC_ARG_CHAOTIC_GOOD_DATA_CONNECTIONS \
+  "grpc.chaotic_good.data_connections"
+
 namespace grpc_core {
 namespace chaotic_good {
 class ChaoticGoodServerListener final : public Server::ListenerInterface {
@@ -109,7 +114,7 @@ class ChaoticGoodServerListener final : public Server::ListenerInterface {
 
    private:
     void Done();
-    void NewConnectionID();
+    void NewConnectionIDs(size_t count);
     RefCountedPtr<Arena> arena_ = SimpleArenaAllocator()->MakeArena();
     const RefCountedPtr<ChaoticGoodServerListener> listener_;
     RefCountedPtr<HandshakingState> handshaking_state_;
@@ -117,9 +122,9 @@ class ChaoticGoodServerListener final : public Server::ListenerInterface {
     ActivityPtr receive_settings_activity_ ABSL_GUARDED_BY(mu_);
     bool orphaned_ ABSL_GUARDED_BY(mu_) = false;
     PromiseEndpoint endpoint_;
-    absl::BitGen bitgen_;
+    std::vector<std::string> connection_ids_;
-    std::string connection_id_;
     int32_t data_alignment_;
+    absl::BitGen bitgen_;
   };
 
   void Start(Server*, const std::vector<grpc_pollset*>*) override {

src/core/ext/transport/chaotic_good/server_transport.cc

@@ -51,25 +51,6 @@
 namespace grpc_core {
 namespace chaotic_good {
 
-auto ChaoticGoodServerTransport::TransportWriteLoop(
-    RefCountedPtr<ChaoticGoodTransport> transport) {
-  return Loop([this, transport = std::move(transport)] {
-    return TrySeq(
-        // Get next outgoing frame.
-        outgoing_frames_.Next(),
-        // Serialize and write it out.
-        [transport = transport.get()](ServerFrame client_frame) {
-          return transport->WriteFrame(GetFrameInterface(client_frame));
-        },
-        []() -> LoopCtl<absl::Status> {
-          // The write failures will be caught in TrySeq and exit loop.
-          // Therefore, only need to return Continue() in the last lambda
-          // function.
-          return Continue();
-        });
-  });
-}
-
 auto ChaoticGoodServerTransport::PushFrameIntoCall(CallInitiator call_initiator,
                                                    MessageFrame frame) {
   GRPC_TRACE_LOG(chaotic_good, INFO)
@@ -86,29 +67,35 @@ auto ChaoticGoodServerTransport::PushFrameIntoCall(CallInitiator call_initiator,
 }
 
 template <typename T>
-auto ChaoticGoodServerTransport::DispatchFrame(ChaoticGoodTransport& transport,
+auto ChaoticGoodServerTransport::DispatchFrame(
-                                               const FrameHeader& header,
+    RefCountedPtr<ChaoticGoodTransport> transport, IncomingFrame frame) {
-                                               SliceBuffer payload) {
+  absl::optional<CallInitiator> call_initiator =
-  return TrySeq(
+      LookupStream(frame.header().stream_id);
-      [&transport, header, payload = std::move(payload)]() mutable {
+  return If(
-        return transport.DeserializeFrame<T>(header, std::move(payload));
+      call_initiator.has_value(),
-      },
+      [this, &call_initiator, &frame, &transport]() {
-      [this](T frame) {
+        // TODO(ctiller): instead of SpawnWaitable here we probably want a
-        absl::optional<CallInitiator> call_initiator =
+        // small queue to push into, so that the call can proceed
-            LookupStream(frame.stream_id);
+        // asynchronously to other calls regardless of frame ordering.
-        return If(
+        return call_initiator->SpawnWaitable(
-            call_initiator.has_value(),
+            "push-frame",
-            [this, &call_initiator, &frame]() {
+            [this, call_initiator = *call_initiator, frame = std::move(frame),
-              return call_initiator->SpawnWaitable(
+             transport = std::move(transport)]() mutable {
-                  "push-frame", [this, call_initiator = *call_initiator,
+              return TrySeq(
-                                 frame = std::move(frame)]() mutable {
+                  frame.Payload(),
+                  [transport = std::move(transport),
+                   header = frame.header()](SliceBuffer payload) {
+                    return transport->DeserializeFrame<T>(header,
+                                                          std::move(payload));
+                  },
+                  [call_initiator, this](T frame) mutable {
                     return Map(call_initiator.CancelIfFails(PushFrameIntoCall(
                                    call_initiator, std::move(frame))),
                                [](StatusFlag) { return absl::OkStatus(); });
                   });
-            },
+            });
-            []() { return absl::OkStatus(); });
+      },
-      });
+      []() { return absl::OkStatus(); });
 }
 
 namespace {
@@ -240,35 +227,39 @@ absl::Status ChaoticGoodServerTransport::NewStream(
   return absl::OkStatus();
 }
 
-auto ChaoticGoodServerTransport::ReadOneFrame(ChaoticGoodTransport& transport) {
+auto ChaoticGoodServerTransport::ReadOneFrame(
+    RefCountedPtr<ChaoticGoodTransport> transport) {
   return GRPC_LATENT_SEE_PROMISE(
       "ReadOneFrame",
       TrySeq(
-          transport.ReadFrameBytes(),
+          transport->ReadFrameBytes(),
-          [this, transport = &transport](
+          [this, transport](IncomingFrame incoming_frame) mutable {
-              std::tuple<FrameHeader, SliceBuffer> frame_bytes) {
+            // CHECK_EQ(header.payload_length, payload.Length());
-            const auto& header = std::get<0>(frame_bytes);
-            SliceBuffer& payload = std::get<1>(frame_bytes);
-            CHECK_EQ(header.payload_length, payload.Length());
             return Switch(
-                header.type,
+                incoming_frame.header().type,
                 Case<FrameType, FrameType::kClientInitialMetadata>([&, this]() {
-                  return Immediate(
+                  return TrySeq(incoming_frame.Payload(),
-                      NewStream(*transport, header, std::move(payload)));
+                                [this, transport = std::move(transport),
+                                 header = incoming_frame.header()](
+                                    SliceBuffer payload) mutable {
+                                  return NewStream(*transport, header,
+                                                   std::move(payload));
+                                });
                 }),
-                Case<FrameType, FrameType::kMessage>([&, this]() {
+                Case<FrameType, FrameType::kMessage>([&, this]() mutable {
-                  return DispatchFrame<MessageFrame>(*transport, header,
+                  return DispatchFrame<MessageFrame>(std::move(transport),
-                                                     std::move(payload));
+                                                     std::move(incoming_frame));
-                }),
-                Case<FrameType, FrameType::kClientEndOfStream>([&, this]() {
-                  return DispatchFrame<ClientEndOfStream>(*transport, header,
-                                                          std::move(payload));
                 }),
+                Case<FrameType, FrameType::kClientEndOfStream>(
+                    [&, this]() mutable {
+                      return DispatchFrame<ClientEndOfStream>(
+                          std::move(transport), std::move(incoming_frame));
+                    }),
                 Case<FrameType, FrameType::kCancel>([&, this]() {
                   absl::optional<CallInitiator> call_initiator =
-                      ExtractStream(header.stream_id);
+                      ExtractStream(incoming_frame.header().stream_id);
                   GRPC_TRACE_LOG(chaotic_good, INFO)
-                      << "Cancel stream " << header.stream_id
+                      << "Cancel stream " << incoming_frame.header().stream_id
                       << (call_initiator.has_value() ? " (active)"
                                                      : " (not found)");
                   return If(
@@ -283,9 +274,8 @@ auto ChaoticGoodServerTransport::ReadOneFrame(ChaoticGoodTransport& transport) {
                       []() -> absl::Status { return absl::OkStatus(); });
                 }),
                 Default([&]() {
-                  return absl::InternalError(
+                  return absl::InternalError(absl::StrCat(
-                      absl::StrCat("Unexpected frame type: ",
+                      "Unexpected frame type: ", incoming_frame.header().type));
-                                   static_cast<uint8_t>(header.type)));
                 }));
           },
           []() -> LoopCtl<absl::Status> { return Continue{}; }));
@@ -294,8 +284,8 @@ auto ChaoticGoodServerTransport::ReadOneFrame(ChaoticGoodTransport& transport) {
 auto ChaoticGoodServerTransport::TransportReadLoop(
     RefCountedPtr<ChaoticGoodTransport> transport) {
   return Seq(got_acceptor_.Wait(),
-             Loop([this, transport = std::move(transport)] {
+             Loop([this, transport = std::move(transport)]() mutable {
-               return ReadOneFrame(*transport);
+               return ReadOneFrame(transport);
              }));
 }
 
@@ -312,7 +302,7 @@ auto ChaoticGoodServerTransport::OnTransportActivityDone(
 
 ChaoticGoodServerTransport::ChaoticGoodServerTransport(
     const ChannelArgs& args, PromiseEndpoint control_endpoint,
-    PromiseEndpoint data_endpoint,
+    std::vector<PromiseEndpoint> data_endpoints,
     std::shared_ptr<grpc_event_engine::experimental::EventEngine> event_engine)
     : call_arena_allocator_(MakeRefCounted<CallArenaAllocator>(
           args.GetObject<ResourceQuota>()
@@ -326,15 +316,17 @@ ChaoticGoodServerTransport::ChaoticGoodServerTransport(
       args.GetInt("grpc.chaotic_good.inlined_payload_size_threshold")
           .value_or(options.inlined_payload_size_threshold);
   auto transport = MakeRefCounted<ChaoticGoodTransport>(
-      std::move(control_endpoint), std::move(data_endpoint), options);
+      std::move(control_endpoint), std::move(data_endpoints), event_engine,
+      options);
   auto party_arena = SimpleArenaAllocator(0)->MakeArena();
   party_arena->SetContext<grpc_event_engine::experimental::EventEngine>(
       event_engine.get());
   party_ = Party::Make(std::move(party_arena));
-  party_->Spawn("server-chaotic-writer",
+  party_->Spawn(
-                GRPC_LATENT_SEE_PROMISE("ServerTransportWriteLoop",
+      "server-chaotic-writer",
-                                        TransportWriteLoop(transport)),
+      GRPC_LATENT_SEE_PROMISE("ServerTransportWriteLoop",
-                OnTransportActivityDone("writer"));
+                              transport->TransportWriteLoop(outgoing_frames_)),
+      OnTransportActivityDone("writer"));
   party_->Spawn("server-chaotic-reader",
                 GRPC_LATENT_SEE_PROMISE("ServerTransportReadLoop",
                                         TransportReadLoop(transport)),

src/core/ext/transport/chaotic_good/server_transport.h

@@ -77,7 +77,7 @@ class ChaoticGoodServerTransport final : public ServerTransport {
  public:
   ChaoticGoodServerTransport(
       const ChannelArgs& args, PromiseEndpoint control_endpoint,
-      PromiseEndpoint data_endpoint,
+      std::vector<PromiseEndpoint> data_endpoints,
       std::shared_ptr<grpc_event_engine::experimental::EventEngine>
           event_engine);
 
@@ -108,8 +108,7 @@ class ChaoticGoodServerTransport final : public ServerTransport {
   auto CallOutboundLoop(uint32_t stream_id, CallInitiator call_initiator);
   auto OnTransportActivityDone(absl::string_view activity);
   auto TransportReadLoop(RefCountedPtr<ChaoticGoodTransport> transport);
-  auto ReadOneFrame(ChaoticGoodTransport& transport);
+  auto ReadOneFrame(RefCountedPtr<ChaoticGoodTransport> transport);
-  auto TransportWriteLoop(RefCountedPtr<ChaoticGoodTransport> transport);
   // Read different parts of the server frame from control/data endpoints
   // based on frame header.
   // Resolves to a StatusOr<tuple<SliceBuffer, SliceBuffer>>
@@ -125,8 +124,8 @@ class ChaoticGoodServerTransport final : public ServerTransport {
                          const FrameHeader& header,
                          SliceBuffer initial_metadata_payload);
   template <typename T>
-  auto DispatchFrame(ChaoticGoodTransport& transport, const FrameHeader& header,
+  auto DispatchFrame(RefCountedPtr<ChaoticGoodTransport> transport,
-                     SliceBuffer payload);
+                     IncomingFrame frame);
   auto PushFrameIntoCall(CallInitiator call_initiator, MessageFrame frame);
   auto PushFrameIntoCall(CallInitiator call_initiator, ClientEndOfStream frame);
   auto SendFrame(ServerFrame frame, MpscSender<ServerFrame> outgoing_frames,

src/core/lib/promise/all_ok.h

@@ -24,6 +24,7 @@
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "src/core/lib/promise/detail/join_state.h"
+#include "src/core/lib/promise/detail/promise_factory.h"
 #include "src/core/lib/promise/map.h"
 #include "src/core/lib/promise/poll.h"
 #include "src/core/lib/promise/status_flag.h"
@@ -81,6 +82,42 @@ GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline auto AllOk(Promises... promises) {
   return promise_detail::AllOk<Result, Promises...>(std::move(promises)...);
 }
 
+// Construct a promise for each element of the set, then run them all.
+// If any fail, cancel the rest and return the failure.
+// If all succeed, return Ok.
+template <typename Result, typename Iter, typename FactoryFn>
+inline auto AllOkIter(Iter begin, Iter end, FactoryFn factory_fn) {
+  using Factory =
+      promise_detail::RepeatedPromiseFactory<decltype(*begin), FactoryFn>;
+  Factory factory(std::move(factory_fn));
+  using Promise = typename Factory::Promise;
+  std::vector<Promise> promises;
+  std::vector<bool> done;
+  for (auto it = begin; it != end; ++it) {
+    promises.emplace_back(factory.Make(*it));
+    done.push_back(false);
+  }
+  return [promises = std::move(promises),
+          done = std::move(done)]() mutable -> Poll<Result> {
+    using Traits = promise_detail::AllOkTraits<Result>;
+    bool still_working = false;
+    for (size_t i = 0; i < promises.size(); ++i) {
+      if (done[i]) continue;
+      auto p = promises[i]();
+      if (auto* r = p.value_if_ready()) {
+        if (!Traits::IsOk(*r)) {
+          return Traits::template EarlyReturn<Result>(std::move(*r));
+        }
+        done[i] = true;
+      } else {
+        still_working = true;
+      }
+    }
+    if (still_working) return Pending{};
+    return Traits::FinalReturn();
+  };
+}
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_PROMISE_ALL_OK_H

src/core/lib/promise/detail/basic_seq.h

@@ -17,35 +17,41 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "src/core/lib/promise/detail/promise_factory.h"
 #include "src/core/lib/promise/poll.h"
 #include "src/core/util/construct_destruct.h"
 
 namespace grpc_core {
 namespace promise_detail {
 
+template <typename FactoryFn>
+auto BindFactoryFnArgs(FactoryFn fn) {
+  return [fn = std::move(fn)](auto x) mutable {
+    return fn(std::get<0>(x), std::move(std::get<1>(x)));
+  };
+}
+
 // Models a sequence of unknown size
 // At each element, the accumulator A and the current value V is passed to some
 // function of type IterTraits::Factory as f(V, IterTraits::Argument); f is
 // expected to return a promise that resolves to IterTraits::Wrapped.
-template <class IterTraits>
+template <template <typename> class Traits, typename Iter, typename FactoryFn,
+          typename Argument>
 class BasicSeqIter {
  private:
-  using Traits = typename IterTraits::Traits;
+  using BoundFactoryFn = decltype(BindFactoryFnArgs(std::declval<FactoryFn>()));
-  using Iter = typename IterTraits::Iter;
+  using TplArg = std::tuple<decltype((*std::declval<Iter>())), Argument>;
-  using Factory = typename IterTraits::Factory;
+  using Factory = RepeatedPromiseFactory<TplArg, BoundFactoryFn>;
-  using Argument = typename IterTraits::Argument;
+  using State = typename Factory::Promise;
-  using IterValue = typename IterTraits::IterValue;
+  using StateResult = typename State::Result;
-  using StateCreated = typename IterTraits::StateCreated;
-  using State = typename IterTraits::State;
-  using Wrapped = typename IterTraits::Wrapped;
 
  public:
-  BasicSeqIter(Iter begin, Iter end, Factory f, Argument arg)
+  BasicSeqIter(Iter begin, Iter end, FactoryFn f, Argument arg)
-      : cur_(begin), end_(end), f_(std::move(f)) {
+      : cur_(begin), end_(end), f_(BindFactoryFnArgs(std::move(f))) {
     if (cur_ == end_) {
       Construct(&result_, std::move(arg));
     } else {
-      Construct(&state_, f_(*cur_, std::move(arg)));
+      Construct(&state_, f_.Make(TplArg(*cur_, std::move(arg))));
     }
   }
 
@@ -79,7 +85,7 @@ class BasicSeqIter {
     return *this;
   }
 
-  Poll<Wrapped> operator()() {
+  Poll<StateResult> operator()() {
     if (cur_ == end_) {
       return std::move(result_);
     }
@@ -87,11 +93,12 @@ class BasicSeqIter {
   }
 
  private:
-  Poll<Wrapped> PollNonEmpty() {
+  Poll<StateResult> PollNonEmpty() {
-    Poll<Wrapped> r = state_();
+    Poll<StateResult> r = state_();
     if (r.pending()) return r;
-    return Traits::template CheckResultAndRunNext<Wrapped>(
+    using Tr = Traits<StateResult>;
-        std::move(r.value()), [this](Wrapped arg) -> Poll<Wrapped> {
+    return Tr::template CheckResultAndRunNext<StateResult>(
+        std::move(r.value()), [this](auto arg) -> Poll<StateResult> {
           auto next = cur_;
           ++next;
           if (next == end_) {
@@ -99,7 +106,14 @@ class BasicSeqIter {
           }
           cur_ = next;
           state_.~State();
-          Construct(&state_, Traits::CallSeqFactory(f_, *cur_, std::move(arg)));
+          struct WrapperFactory {
+            BasicSeqIter* owner;
+            State Make(typename Tr::UnwrappedType r) {
+              return owner->f_.Make(TplArg(*owner->cur_, std::move(r)));
+            }
+          };
+          WrapperFactory wrapper_factory{this};
+          Construct(&state_, Tr::CallFactory(&wrapper_factory, std::move(arg)));
           return PollNonEmpty();
         });
   }

src/core/lib/promise/join.h

@@ -22,6 +22,7 @@
 
 #include "absl/meta/type_traits.h"
 #include "src/core/lib/promise/detail/join_state.h"
+#include "src/core/lib/promise/detail/promise_factory.h"
 #include "src/core/lib/promise/map.h"
 
 namespace grpc_core {
@@ -84,6 +85,41 @@ GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION inline auto Join(F promise) {
   return Map(std::move(promise), promise_detail::WrapInTuple{});
 }
 
+template <typename Iter, typename FactoryFn>
+inline auto JoinIter(Iter begin, Iter end, FactoryFn factory_fn) {
+  using Factory =
+      promise_detail::RepeatedPromiseFactory<decltype(*begin), FactoryFn>;
+  Factory factory(std::move(factory_fn));
+  using Promise = typename Factory::Promise;
+  using Result = typename Promise::Result;
+  using State = absl::variant<Promise, Result>;
+  std::vector<State> state;
+  for (Iter it = begin; it != end; ++it) {
+    state.emplace_back(factory.Make(*it));
+  }
+  return [state = std::move(state)]() mutable -> Poll<std::vector<Result>> {
+    bool still_working = false;
+    for (auto& s : state) {
+      if (auto* promise = absl::get_if<Promise>(&s)) {
+        auto p = (*promise)();
+        if (auto* r = p.value_if_ready()) {
+          s.template emplace<Result>(std::move(*r));
+        } else {
+          still_working = true;
+        }
+      }
+    }
+    if (!still_working) {
+      std::vector<Result> output;
+      for (auto& s : state) {
+        output.emplace_back(std::move(absl::get<Result>(s)));
+      }
+      return output;
+    }
+    return Pending{};
+  };
+}
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_PROMISE_JOIN_H

src/core/lib/promise/match_promise.h

@@ -53,7 +53,7 @@ struct ConstructPromiseVariantVisitor {
   template <typename T>
   auto operator()(T x) -> absl::variant<promise_detail::PromiseLike<
       decltype(CallConstructorThenFactory(std::declval<Ts>()))>...> {
-    return CallConstructorThenFactory(x);
+    return CallConstructorThenFactory(std::move(x));
   }
 };
 

src/core/lib/promise/party.cc

@@ -251,13 +251,16 @@ void Party::RunLockedAndUnref(Party* party, uint64_t prev_state) {
       // gets held for a really long time.
       auto wakeup =
           std::exchange(g_run_state->next, PartyWakeup{party, prev_state});
-      party->arena_->GetContext<grpc_event_engine::experimental::EventEngine>()
+      auto arena = party->arena_.get();
-          ->Run([wakeup]() {
+      auto* event_engine =
-            GRPC_LATENT_SEE_PARENT_SCOPE("Party::RunLocked offload");
+          arena->GetContext<grpc_event_engine::experimental::EventEngine>();
-            ApplicationCallbackExecCtx app_exec_ctx;
+      CHECK(event_engine != nullptr) << "; " << GRPC_DUMP_ARGS(party, arena);
-            ExecCtx exec_ctx;
+      event_engine->Run([wakeup]() {
-            RunState{wakeup}.Run();
+        GRPC_LATENT_SEE_PARENT_SCOPE("Party::RunLocked offload");
-          });
+        ApplicationCallbackExecCtx app_exec_ctx;
+        ExecCtx exec_ctx;
+        RunState{wakeup}.Run();
+      });
       return;
     }
     g_run_state->next = PartyWakeup{party, prev_state};

src/core/lib/promise/poll.h

@@ -286,6 +286,19 @@ void AbslStringify(Sink& sink, const Poll<T>& poll) {
   absl::Format(&sink, "%v", poll.value());
 }
 
+template <typename Sink, typename T>
+void AbslStringify(Sink& sink, const Poll<absl::StatusOr<T>>& poll) {
+  if (poll.pending()) {
+    absl::Format(&sink, "<<pending>>");
+    return;
+  }
+  if (poll.value().ok()) {
+    absl::Format(&sink, "%v", *poll.value());
+  } else {
+    absl::Format(&sink, "%v", poll.value().status());
+  }
+}
+
 template <typename Sink, typename T>
 void AbslStringify(Sink& sink, const Poll<absl::optional<T>>& poll) {
   if (poll.pending()) {

src/core/lib/promise/seq.h

@@ -47,12 +47,6 @@ struct SeqTraits {
   static R ReturnValue(T&&) {
     abort();
   }
-  template <typename F, typename Elem>
-  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallSeqFactory(F& f,
-                                                                  Elem&& elem,
-                                                                  T&& value) {
-    return f(std::forward<Elem>(elem), std::forward<T>(value));
-  }
   template <typename Result, typename PriorResult, typename RunNext>
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
   CheckResultAndRunNext(PriorResult prior, RunNext run_next) {
@@ -77,25 +71,8 @@ class Seq {
   SeqState<SeqTraits, P, Fs...> state_;
 };
 
-template <typename I, typename F, typename Arg>
-struct SeqIterTraits {
-  using Iter = I;
-  using Factory = F;
-  using Argument = Arg;
-  using IterValue = decltype(*std::declval<Iter>());
-  using StateCreated = decltype(std::declval<F>()(std::declval<IterValue>(),
-                                                  std::declval<Arg>()));
-  using State = PromiseLike<StateCreated>;
-  using Wrapped = typename State::Result;
-
-  using Traits = SeqTraits<Wrapped>;
-};
-
 template <typename Iter, typename Factory, typename Argument>
-struct SeqIterResultTraits {
+using SeqIter = BasicSeqIter<SeqTraits, Iter, Factory, Argument>;
-  using IterTraits = SeqIterTraits<Iter, Factory, Argument>;
-  using Result = BasicSeqIter<IterTraits>;
-};
 
 }  // namespace promise_detail
 
@@ -227,11 +204,11 @@ GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
 //   }
 //   return argument;
 template <typename Iter, typename Factory, typename Argument>
-typename promise_detail::SeqIterResultTraits<Iter, Factory, Argument>::Result
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto SeqIter(Iter begin, Iter end,
-SeqIter(Iter begin, Iter end, Argument argument, Factory factory) {
+                                                  Argument argument,
-  using Result = typename promise_detail::SeqIterResultTraits<Iter, Factory,
+                                                  Factory factory) {
-                                                              Argument>::Result;
+  return promise_detail::SeqIter<Iter, Factory, Argument>(
-  return Result(begin, end, std::move(factory), std::move(argument));
+      begin, end, std::move(factory), std::move(argument));
 }
 
 }  // namespace grpc_core

src/core/lib/promise/try_seq.h

@@ -53,13 +53,6 @@ struct TrySeqTraitsWithSfinae {
   static R ReturnValue(T&&) {
     abort();
   }
-  template <typename F, typename Elem>
-  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallSeqFactory(F& f,
-                                                                  Elem&& elem,
-                                                                  T&& value)
-      -> decltype(f(std::forward<Elem>(elem), std::forward<T>(value))) {
-    return f(std::forward<Elem>(elem), std::forward<T>(value));
-  }
   template <typename Result, typename RunNext>
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
   CheckResultAndRunNext(T prior, RunNext run_next) {
@@ -88,12 +81,6 @@ struct TrySeqTraitsWithSfinae<absl::StatusOr<T>> {
       absl::StatusOr<T>&& status) {
     return FailureStatusCast<R>(status.status());
   }
-  template <typename F, typename Elem>
-  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static auto CallSeqFactory(
-      F& f, Elem&& elem, absl::StatusOr<T> value)
-      -> decltype(f(std::forward<Elem>(elem), std::move(*value))) {
-    return f(std::forward<Elem>(elem), std::move(*value));
-  }
   template <typename Result, typename RunNext>
   GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static Poll<Result>
   CheckResultAndRunNext(absl::StatusOr<T> prior, RunNext run_next) {
@@ -241,24 +228,30 @@ class TrySeq {
   SeqState<TrySeqTraits, P, Fs...> state_;
 };
 
-template <typename I, typename F, typename Arg>
-struct TrySeqIterTraits {
-  using Iter = I;
-  using Factory = F;
-  using Argument = Arg;
-  using IterValue = decltype(*std::declval<Iter>());
-  using StateCreated = decltype(std::declval<F>()(std::declval<IterValue>(),
-                                                  std::declval<Arg>()));
-  using State = PromiseLike<StateCreated>;
-  using Wrapped = typename State::Result;
-
-  using Traits = TrySeqTraits<Wrapped>;
-};
-
 template <typename Iter, typename Factory, typename Argument>
-struct TrySeqIterResultTraits {
+using TrySeqIter = BasicSeqIter<TrySeqTraits, Iter, Factory, Argument>;
-  using IterTraits = TrySeqIterTraits<Iter, Factory, Argument>;
+
-  using Result = BasicSeqIter<IterTraits>;
+template <typename Container, typename Factory, typename Argument>
+struct TrySeqContainerResultTraits {
+  using BaseResult =
+      TrySeqIter<typename Container::iterator, Factory, Argument>;
+  class Result {
+   public:
+    Result(Container container, Factory factory, Argument argument)
+        : container_(std::move(container)),
+          base_result_(container_.begin(), container_.end(), std::move(factory),
+                       std::move(argument)) {}
+    Result(const Result&) = delete;
+    Result& operator=(const Result&) = delete;
+    Result(Result&&) = default;
+    Result& operator=(Result&&) = default;
+
+    auto operator()() { return base_result_(); }
+
+   private:
+    Container container_;
+    BaseResult base_result_;
+  };
 };
 
 }  // namespace promise_detail
@@ -352,14 +345,19 @@ GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
 //   }
 //   return argument;
 template <typename Iter, typename Factory, typename Argument>
-GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION auto TrySeqIter(Iter begin, Iter end,
-    typename promise_detail::TrySeqIterResultTraits<Iter, Factory,
+                                                     Argument argument,
-                                                    Argument>::Result
+                                                     Factory factory) {
-    TrySeqIter(Iter begin, Iter end, Argument argument, Factory factory) {
+  return promise_detail::TrySeqIter<Iter, Factory, Argument>(
+      begin, end, std::move(factory), std::move(argument));
+}
+
+template <typename Container, typename Factory, typename Argument>
+auto TrySeqContainer(Container container, Argument argument, Factory factory) {
   using Result =
-      typename promise_detail::TrySeqIterResultTraits<Iter, Factory,
+      typename promise_detail::TrySeqContainerResultTraits<Container, Factory,
-                                                      Argument>::Result;
+                                                           Argument>::Result;
-  return Result(begin, end, std::move(factory), std::move(argument));
+  return Result(std::move(container), std::move(factory), std::move(argument));
 }
 
 }  // namespace grpc_core

test/core/call/yodel/yodel_test.cc

@@ -55,9 +55,9 @@ absl::string_view ActionState::StateString(State state) {
 }
 
 void ActionState::Set(State state, SourceLocation whence) {
-  LOG(INFO) << StateString(state) << " " << name() << " [" << step() << "] "
+  LOG(INFO) << StateString(state) << " " << name() << " [" << step()
-            << file() << ":" << line() << " @ " << whence.file() << ":"
+            << "] t=" << Timestamp::Now() << " " << file() << ":" << line()
-            << whence.line();
+            << " @ " << whence.file() << ":" << whence.line();
   state_ = state;
 }
 
@@ -207,7 +207,8 @@ void YodelTest::WaitForAllPendingWork() {
 
 void YodelTest::Timeout() {
   std::vector<std::string> lines;
-  lines.emplace_back("Timeout waiting for pending actions to complete");
+  lines.emplace_back(absl::StrCat(
+      "Timeout waiting for pending actions to complete ", Timestamp::Now()));
   while (!pending_actions_.empty()) {
     auto action = std::move(pending_actions_.front());
     pending_actions_.pop();

test/core/call/yodel/yodel_test.h

@@ -362,6 +362,35 @@ class YodelTest : public ::testing::Test {
         .Start(std::move(actions)...);
   }
 
+  class NoContext {
+   public:
+    template <typename PromiseFactory>
+    void SpawnInfallible(absl::string_view name,
+                         PromiseFactory promise_factory) {
+      party_->Spawn(
+          name,
+          [party = party_,
+           promise_factory = std::move(promise_factory)]() mutable {
+            promise_detail::OncePromiseFactory<void, PromiseFactory> factory(
+                std::move(promise_factory));
+            return [party, underlying = factory.Make()]() mutable {
+              return underlying();
+            };
+          },
+          [](Empty) {});
+    }
+
+   private:
+    RefCountedPtr<Party> party_ =
+        Party::Make(SimpleArenaAllocator()->MakeArena());
+  };
+
+  template <typename... Actions>
+  void SpawnTestSeqWithoutContext(
+      yodel_detail::NameAndLocation name_and_location, Actions... actions) {
+    SpawnTestSeq(NoContext{}, name_and_location, std::move(actions)...);
+  }
+
   auto MakeCall(ClientMetadataHandle client_initial_metadata) {
     auto arena = state_->call_arena_allocator->MakeArena();
     arena->SetContext<grpc_event_engine::experimental::EventEngine>(

test/core/end2end/end2end_test_suites.cc

@@ -516,11 +516,12 @@ class FixtureWithTracing final : public CoreTestFixture {
   std::unique_ptr<CoreTestFixture> fixture_;
 };
 
-class ChaoticGoodFixture final : public CoreTestFixture {
+class ChaoticGoodFixture : public CoreTestFixture {
  public:
-  explicit ChaoticGoodFixture(std::string localaddr = JoinHostPort(
+  explicit ChaoticGoodFixture(int data_connections = 1,
+                              std::string localaddr = JoinHostPort(
                                   "localhost", grpc_pick_unused_port_or_die()))
-      : localaddr_(std::move(localaddr)) {}
+      : data_connections_(data_connections), localaddr_(std::move(localaddr)) {}
 
  protected:
   const std::string& localaddr() const { return localaddr_; }
@@ -529,7 +530,11 @@ class ChaoticGoodFixture final : public CoreTestFixture {
   grpc_server* MakeServer(
       const ChannelArgs& args, grpc_completion_queue* cq,
       absl::AnyInvocable<void(grpc_server*)>& pre_server_start) override {
-    auto* server = grpc_server_create(args.ToC().get(), nullptr);
+    auto* server = grpc_server_create(
+        args.Set(GRPC_ARG_CHAOTIC_GOOD_DATA_CONNECTIONS, data_connections_)
+            .ToC()
+            .get(),
+        nullptr);
     grpc_server_register_completion_queue(server, cq, nullptr);
     CHECK(grpc_server_add_chaotic_good_port(server, localaddr_.c_str()));
     pre_server_start(server);
@@ -545,9 +550,20 @@ class ChaoticGoodFixture final : public CoreTestFixture {
     return client;
   }
 
+  int data_connections_;
   std::string localaddr_;
 };
 
+class ChaoticGoodSingleConnectionFixture final : public ChaoticGoodFixture {
+ public:
+  ChaoticGoodSingleConnectionFixture() : ChaoticGoodFixture(1) {}
+};
+
+class ChaoticGoodManyConnectionFixture final : public ChaoticGoodFixture {
+ public:
+  ChaoticGoodManyConnectionFixture() : ChaoticGoodFixture(16) {}
+};
+
 #ifdef GRPC_POSIX_WAKEUP_FD
 class InsecureFixtureWithPipeForWakeupFd : public InsecureFixture {
  public:

test/core/promise/BUILD

@@ -313,7 +313,10 @@ grpc_cc_test(
     tags = ["promise_test"],
     uses_event_engine = False,
     uses_polling = False,
-    deps = ["//src/core:try_seq"],
+    deps = [
+        "poll_matcher",
+        "//src/core:try_seq",
+    ],
 )
 
 grpc_cc_test(

test/core/promise/try_seq_test.cc

@@ -16,10 +16,12 @@
 
 #include <stdlib.h>
 
+#include <memory>
 #include <string>
 #include <vector>
 
 #include "gtest/gtest.h"
+#include "test/core/promise/poll_matcher.h"
 
 namespace grpc_core {
 
@@ -183,6 +185,22 @@ TEST(TrySeqIterTest, ErrorAt3) {
             Poll<absl::StatusOr<int>>(absl::CancelledError()));
 }
 
+TEST(TrySeqContainer, Ok) {
+  std::vector<std::unique_ptr<int>> v;
+  v.emplace_back(std::make_unique<int>(1));
+  v.emplace_back(std::make_unique<int>(2));
+  v.emplace_back(std::make_unique<int>(3));
+  int expect = 1;
+  auto p = TrySeqContainer(std::move(v), Empty{},
+                           [&expect](const std::unique_ptr<int>& i, Empty) {
+                             EXPECT_EQ(*i, expect);
+                             ++expect;
+                             return Empty{};
+                           });
+  EXPECT_THAT(p(), IsReady());
+  EXPECT_EQ(expect, 4);
+}
+
 }  // namespace grpc_core
 
 int main(int argc, char** argv) {

test/core/transport/benchmarks/bm_chaotic_good.cc

@@ -40,11 +40,13 @@ class ChaoticGoodTraits {
         MakePassthroughEndpoint(3, 4, true);
     auto client = MakeOrphanable<chaotic_good::ChaoticGoodClientTransport>(
         PromiseEndpoint(std::move(control.client), SliceBuffer()),
-        PromiseEndpoint(std::move(data.client), SliceBuffer()), channel_args,
+        chaotic_good::OneDataEndpoint(
-        grpc_event_engine::experimental::GetDefaultEventEngine());
+            PromiseEndpoint(std::move(data.client), SliceBuffer())),
+        channel_args, grpc_event_engine::experimental::GetDefaultEventEngine());
     auto server = MakeOrphanable<chaotic_good::ChaoticGoodServerTransport>(
         channel_args, PromiseEndpoint(std::move(control.server), SliceBuffer()),
-        PromiseEndpoint(std::move(data.server), SliceBuffer()),
+        chaotic_good::OneDataEndpoint(
+            PromiseEndpoint(std::move(data.server), SliceBuffer())),
         grpc_event_engine::experimental::GetDefaultEventEngine());
     return {std::move(client), std::move(server)};
   }

test/core/transport/chaotic_good/BUILD

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_package")
+load("//test/core/call/yodel:grpc_yodel_test.bzl", "grpc_yodel_simple_test")
 load("//test/core/test_util:grpc_fuzzer.bzl", "grpc_fuzzer", "grpc_proto_fuzzer")
 
 licenses(["notice"])
@@ -243,3 +244,33 @@ grpc_cc_test(
         "//test/core/test_util:grpc_test_util",
     ],
 )
+
+grpc_yodel_simple_test(
+    name = "control_endpoint",
+    srcs = ["control_endpoint_test.cc"],
+    external_deps = ["gtest"],
+    language = "C++",
+    tags = [
+        "no_windows",
+    ],
+    deps = [
+        "//src/core:chaotic_good_control_endpoint",
+        "//test/core/call/yodel:yodel_test",
+        "//test/core/transport/util:mock_promise_endpoint",
+    ],
+)
+
+grpc_yodel_simple_test(
+    name = "data_endpoints",
+    srcs = ["data_endpoints_test.cc"],
+    external_deps = ["gtest"],
+    language = "C++",
+    tags = [
+        "no_windows",
+    ],
+    deps = [
+        "//src/core:chaotic_good_data_endpoints",
+        "//test/core/call/yodel:yodel_test",
+        "//test/core/transport/util:mock_promise_endpoint",
+    ],
+)

test/core/transport/chaotic_good/client_transport_error_test.cc

@@ -186,8 +186,8 @@ TEST_F(ClientTransportTest, AddOneStreamWithWriteFailed) {
   EXPECT_CALL(*control_endpoint.endpoint, Read).WillOnce(Return(false));
   auto transport = MakeOrphanable<ChaoticGoodClientTransport>(
       std::move(control_endpoint.promise_endpoint),
-      std::move(data_endpoint.promise_endpoint), MakeChannelArgs(),
+      OneDataEndpoint(std::move(data_endpoint.promise_endpoint)),
-      event_engine());
+      MakeChannelArgs(), event_engine());
   auto call = MakeCall(TestInitialMetadata());
   transport->StartCall(call.handler.StartCall());
   call.initiator.SpawnGuarded("test-send",
@@ -214,6 +214,8 @@ TEST_F(ClientTransportTest, AddOneStreamWithWriteFailed) {
       });
   // Wait until ClientTransport's internal activities to finish.
   event_engine()->TickUntilIdle();
+  transport.reset();
+  event_engine()->TickUntilIdle();
   event_engine()->UnsetGlobalHooks();
 }
 
@@ -230,8 +232,8 @@ TEST_F(ClientTransportTest, AddOneStreamWithReadFailed) {
           }));
   auto transport = MakeOrphanable<ChaoticGoodClientTransport>(
       std::move(control_endpoint.promise_endpoint),
-      std::move(data_endpoint.promise_endpoint), MakeChannelArgs(),
+      OneDataEndpoint(std::move(data_endpoint.promise_endpoint)),
-      event_engine());
+      MakeChannelArgs(), event_engine());
   auto call = MakeCall(TestInitialMetadata());
   transport->StartCall(call.handler.StartCall());
   call.initiator.SpawnGuarded("test-send",
@@ -258,6 +260,8 @@ TEST_F(ClientTransportTest, AddOneStreamWithReadFailed) {
       });
   // Wait until ClientTransport's internal activities to finish.
   event_engine()->TickUntilIdle();
+  transport.reset();
+  event_engine()->TickUntilIdle();
   event_engine()->UnsetGlobalHooks();
 }
 
@@ -282,8 +286,8 @@ TEST_F(ClientTransportTest, AddMultipleStreamWithWriteFailed) {
   EXPECT_CALL(*control_endpoint.endpoint, Read).WillOnce(Return(false));
   auto transport = MakeOrphanable<ChaoticGoodClientTransport>(
       std::move(control_endpoint.promise_endpoint),
-      std::move(data_endpoint.promise_endpoint), MakeChannelArgs(),
+      OneDataEndpoint(std::move(data_endpoint.promise_endpoint)),
-      event_engine());
+      MakeChannelArgs(), event_engine());
   auto call1 = MakeCall(TestInitialMetadata());
   transport->StartCall(call1.handler.StartCall());
   auto call2 = MakeCall(TestInitialMetadata());
@@ -334,6 +338,8 @@ TEST_F(ClientTransportTest, AddMultipleStreamWithWriteFailed) {
       });
   // Wait until ClientTransport's internal activities to finish.
   event_engine()->TickUntilIdle();
+  transport.reset();
+  event_engine()->TickUntilIdle();
   event_engine()->UnsetGlobalHooks();
 }
 
@@ -350,8 +356,8 @@ TEST_F(ClientTransportTest, AddMultipleStreamWithReadFailed) {
           }));
   auto transport = MakeOrphanable<ChaoticGoodClientTransport>(
       std::move(control_endpoint.promise_endpoint),
-      std::move(data_endpoint.promise_endpoint), MakeChannelArgs(),
+      OneDataEndpoint(std::move(data_endpoint.promise_endpoint)),
-      event_engine());
+      MakeChannelArgs(), event_engine());
   auto call1 = MakeCall(TestInitialMetadata());
   transport->StartCall(call1.handler.StartCall());
   auto call2 = MakeCall(TestInitialMetadata());
@@ -402,6 +408,8 @@ TEST_F(ClientTransportTest, AddMultipleStreamWithReadFailed) {
       });
   // Wait until ClientTransport's internal activities to finish.
   event_engine()->TickUntilIdle();
+  transport.reset();
+  event_engine()->TickUntilIdle();
   event_engine()->UnsetGlobalHooks();
 }
 

test/core/transport/chaotic_good/client_transport_test.cc

@@ -118,8 +118,8 @@ TEST_F(TransportTest, AddOneStream) {
       .WillOnce(Return(false));
   auto transport = MakeOrphanable<ChaoticGoodClientTransport>(
       std::move(control_endpoint.promise_endpoint),
-      std::move(data_endpoint.promise_endpoint), MakeChannelArgs(),
+      OneDataEndpoint(std::move(data_endpoint.promise_endpoint)),
-      event_engine());
+      MakeChannelArgs(), event_engine());
   auto call = MakeCall(TestInitialMetadata());
   StrictMock<MockFunction<void()>> on_done;
   EXPECT_CALL(on_done, Call());
@@ -207,8 +207,8 @@ TEST_F(TransportTest, AddOneStreamMultipleMessages) {
       .WillOnce(Return(false));
   auto transport = MakeOrphanable<ChaoticGoodClientTransport>(
       std::move(control_endpoint.promise_endpoint),
-      std::move(data_endpoint.promise_endpoint), MakeChannelArgs(),
+      OneDataEndpoint(std::move(data_endpoint.promise_endpoint)),
-      event_engine());
+      MakeChannelArgs(), event_engine());
   auto call = MakeCall(TestInitialMetadata());
   StrictMock<MockFunction<void()>> on_done;
   EXPECT_CALL(on_done, Call());

test/core/transport/chaotic_good/control_endpoint_test.cc

@@ -0,0 +1,45 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/core/ext/transport/chaotic_good/control_endpoint.h"
+
+#include <grpc/grpc.h>
+
+#include "gtest/gtest.h"
+#include "test/core/call/yodel/yodel_test.h"
+#include "test/core/transport/util/mock_promise_endpoint.h"
+
+namespace grpc_core {
+
+class ControlEndpointTest : public YodelTest {
+ protected:
+  using YodelTest::YodelTest;
+};
+
+#define CONTROL_ENDPOINT_TEST(name) YODEL_TEST(ControlEndpointTest, name)
+
+CONTROL_ENDPOINT_TEST(CanWrite) {
+  chaotic_good::testing::MockPromiseEndpoint ep(1234);
+  chaotic_good::ControlEndpoint control_endpoint(std::move(ep.promise_endpoint),
+                                                 event_engine().get());
+  ep.ExpectWrite(
+      {grpc_event_engine::experimental::Slice::FromCopiedString("hello")},
+      nullptr);
+  SpawnTestSeqWithoutContext(
+      "write",
+      control_endpoint.Write(SliceBuffer(Slice::FromCopiedString("hello"))));
+  WaitForAllPendingWork();
+}
+
+}  // namespace grpc_core

test/core/transport/chaotic_good/data_endpoints_test.cc

@@ -0,0 +1,119 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/core/ext/transport/chaotic_good/data_endpoints.h"
+
+#include <grpc/grpc.h>
+
+#include "gtest/gtest.h"
+#include "test/core/call/yodel/yodel_test.h"
+#include "test/core/transport/util/mock_promise_endpoint.h"
+
+namespace grpc_core {
+
+class DataEndpointsTest : public YodelTest {
+ protected:
+  using YodelTest::YodelTest;
+};
+
+#define DATA_ENDPOINTS_TEST(name) YODEL_TEST(DataEndpointsTest, name)
+
+DATA_ENDPOINTS_TEST(CanWrite) {
+  chaotic_good::testing::MockPromiseEndpoint ep(1234);
+  std::vector<PromiseEndpoint> endpoints;
+  endpoints.push_back(std::move(ep.promise_endpoint));
+  chaotic_good::DataEndpoints data_endpoints(std::move(endpoints),
+                                             event_engine().get());
+  ep.ExpectWrite(
+      {grpc_event_engine::experimental::Slice::FromCopiedString("hello")},
+      event_engine().get());
+  SpawnTestSeqWithoutContext(
+      "write",
+      data_endpoints.Write(SliceBuffer(Slice::FromCopiedString("hello"))),
+      [](uint32_t id) {
+        EXPECT_EQ(id, 0);
+        return Empty{};
+      });
+  WaitForAllPendingWork();
+}
+
+DATA_ENDPOINTS_TEST(CanMultiWrite) {
+  chaotic_good::testing::MockPromiseEndpoint ep1(1234);
+  chaotic_good::testing::MockPromiseEndpoint ep2(1235);
+  std::vector<PromiseEndpoint> endpoints;
+  endpoints.push_back(std::move(ep1.promise_endpoint));
+  endpoints.push_back(std::move(ep2.promise_endpoint));
+  chaotic_good::DataEndpoints data_endpoints(std::move(endpoints),
+                                             event_engine().get());
+  SliceBuffer writes1;
+  SliceBuffer writes2;
+  ep1.CaptureWrites(writes1, event_engine().get());
+  ep2.CaptureWrites(writes2, event_engine().get());
+  uint32_t write1_ep = 42;
+  uint32_t write2_ep = 42;
+  SpawnTestSeqWithoutContext(
+      "write",
+      data_endpoints.Write(SliceBuffer(Slice::FromCopiedString("hello"))),
+      [&write1_ep](uint32_t id) {
+        write1_ep = id;
+        return Empty{};
+      },
+      data_endpoints.Write(SliceBuffer(Slice::FromCopiedString("world"))),
+      [&write2_ep](uint32_t id) {
+        write2_ep = id;
+        return Empty{};
+      });
+  WaitForAllPendingWork();
+  EXPECT_THAT(write1_ep, ::testing::AnyOf(0, 1));
+  EXPECT_THAT(write2_ep, ::testing::AnyOf(0, 1));
+  std::string expect[2];
+  expect[write1_ep] += "hello";
+  expect[write2_ep] += "world";
+  EXPECT_EQ(writes1.JoinIntoString(), expect[0]);
+  EXPECT_EQ(writes2.JoinIntoString(), expect[1]);
+}
+
+DATA_ENDPOINTS_TEST(CanRead) {
+  chaotic_good::testing::MockPromiseEndpoint ep(1234);
+  std::vector<PromiseEndpoint> endpoints;
+  endpoints.push_back(std::move(ep.promise_endpoint));
+  chaotic_good::DataEndpoints data_endpoints(std::move(endpoints),
+                                             event_engine().get());
+  ep.ExpectRead(
+      {grpc_event_engine::experimental::Slice::FromCopiedString("hello")},
+      event_engine().get());
+  SpawnTestSeqWithoutContext("read", data_endpoints.Read(0, 5).Await(),
+                             [](absl::StatusOr<SliceBuffer> result) {
+                               EXPECT_TRUE(result.ok());
+                               EXPECT_EQ(result->JoinIntoString(), "hello");
+                               return Empty{};
+                             });
+  WaitForAllPendingWork();
+}
+
+}  // namespace grpc_core

test/core/transport/chaotic_good/server_transport_test.cc

@@ -89,7 +89,8 @@ TEST_F(TransportTest, ReadAndWriteOneMessage) {
           .channel_args_preconditioning()
           .PreconditionChannelArgs(nullptr),
       std::move(control_endpoint.promise_endpoint),
-      std::move(data_endpoint.promise_endpoint), event_engine());
+      OneDataEndpoint(std::move(data_endpoint.promise_endpoint)),
+      event_engine());
   const auto server_initial_metadata =
       EncodeProto<chaotic_good_frame::ServerMetadata>("message: 'hello'");
   const auto server_trailing_metadata =
@@ -172,10 +173,8 @@ TEST_F(TransportTest, ReadAndWriteOneMessage) {
   control_endpoint.ExpectWrite(
       {SerializedFrameHeader(FrameType::kServerInitialMetadata, 0, 1,
                              server_initial_metadata.length()),
-       server_initial_metadata.Copy()},
+       server_initial_metadata.Copy(),
-      nullptr);
+       SerializedFrameHeader(FrameType::kMessage, 0, 1, 8),
-  control_endpoint.ExpectWrite(
-      {SerializedFrameHeader(FrameType::kMessage, 0, 1, 8),
        EventEngineSlice::FromCopiedString("87654321")},
       nullptr);
   control_endpoint.ExpectWrite(

test/core/transport/test_suite/chaotic_good_fixture.cc

@@ -103,12 +103,36 @@ TRANSPORT_FIXTURE(ChaoticGood) {
           .SetObject(std::static_pointer_cast<EventEngine>(event_engine));
   auto client_transport =
       MakeOrphanable<chaotic_good::ChaoticGoodClientTransport>(
-          std::move(control_endpoints.client), std::move(data_endpoints.client),
+          std::move(control_endpoints.client),
+          chaotic_good::OneDataEndpoint(std::move(data_endpoints.client)),
           ChannelArgs().SetObject(resource_quota), event_engine);
   auto server_transport =
       MakeOrphanable<chaotic_good::ChaoticGoodServerTransport>(
           channel_args, std::move(control_endpoints.server),
-          std::move(data_endpoints.server), event_engine);
+          chaotic_good::OneDataEndpoint(std::move(data_endpoints.server)),
+          event_engine);
+  return ClientAndServerTransportPair{std::move(client_transport),
+                                      std::move(server_transport)};
+}
+
+TRANSPORT_FIXTURE(ChaoticGoodSingleConnection) {
+  auto resource_quota = MakeResourceQuota("test");
+  EndpointPair control_endpoints =
+      CreateEndpointPair(event_engine.get(), resource_quota, 1234);
+  EndpointPair data_endpoints =
+      CreateEndpointPair(event_engine.get(), resource_quota, 4321);
+  auto channel_args =
+      ChannelArgs()
+          .SetObject(resource_quota)
+          .SetObject(std::static_pointer_cast<EventEngine>(event_engine));
+  auto client_transport =
+      MakeOrphanable<chaotic_good::ChaoticGoodClientTransport>(
+          std::move(control_endpoints.client), std::vector<PromiseEndpoint>{},
+          ChannelArgs().SetObject(resource_quota), event_engine);
+  auto server_transport =
+      MakeOrphanable<chaotic_good::ChaoticGoodServerTransport>(
+          channel_args, std::move(control_endpoints.server),
+          std::vector<PromiseEndpoint>{}, event_engine);
   return ClientAndServerTransportPair{std::move(client_transport),
                                       std::move(server_transport)};
 }

test/core/transport/util/mock_promise_endpoint.cc

@@ -84,6 +84,30 @@ void MockPromiseEndpoint::ExpectWrite(
           }));
 }
 
+void MockPromiseEndpoint::CaptureWrites(SliceBuffer& writes,
+                                        EventEngine* schedule_on_event_engine) {
+  EXPECT_CALL(*endpoint, Write)
+      .Times(::testing::AtLeast(0))
+      .WillRepeatedly(WithArgs<0, 1>(
+          [writes = &writes, schedule_on_event_engine](
+              absl::AnyInvocable<void(absl::Status)> on_writable,
+              grpc_event_engine::experimental::SliceBuffer* buffer) {
+            SliceBuffer temp;
+            grpc_slice_buffer_swap(temp.c_slice_buffer(),
+                                   buffer->c_slice_buffer());
+            writes->Append(temp);
+            if (schedule_on_event_engine != nullptr) {
+              schedule_on_event_engine->Run(
+                  [on_writable = std::move(on_writable)]() mutable {
+                    on_writable(absl::OkStatus());
+                  });
+              return false;
+            } else {
+              return true;
+            }
+          }));
+}
+
 }  // namespace testing
 }  // namespace chaotic_good
 }  // namespace grpc_core

test/core/transport/util/mock_promise_endpoint.h

@@ -81,6 +81,9 @@ struct MockPromiseEndpoint {
   void ExpectWrite(
       std::initializer_list<grpc_event_engine::experimental::Slice> slices,
       grpc_event_engine::experimental::EventEngine* schedule_on_event_engine);
+  void CaptureWrites(
+      SliceBuffer& writes,
+      grpc_event_engine::experimental::EventEngine* schedule_on_event_engine);
 };
 
 }  // namespace testing

tools/distrib/fix_build_deps.py

@@ -445,6 +445,8 @@ for dirname in [
             "grpc_fuzz_test": grpc_cc_library,
             "grpc_proto_fuzzer": grpc_cc_library,
             "grpc_proto_library": grpc_proto_library,
+            "grpc_internal_proto_library": grpc_proto_library,
+            "grpc_cc_proto_library": grpc_cc_library,
             "select": lambda d: d["//conditions:default"],
             "glob": lambda files, **kwargs: None,
             "grpc_end2end_tests": lambda: None,
@@ -459,6 +461,7 @@ for dirname in [
             "grpc_clang_cl_settings": lambda **kwargs: None,
             "grpc_benchmark_args": lambda **kwargs: [],
             "LARGE_MACHINE": 1,
+            "HISTORY": 1,
         },
         {},
     )