[test] Add comment about workaround (re #35969) (#35971)

Followup from #35969

This also fixes a few scenarios we had not caught earlier.

Closes #35971

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/35971 from drfloob:doc 32de2d7033
PiperOrigin-RevId: 609509072

test/core/end2end/tests/cancel_after_invoke.cc

@@ -38,6 +38,9 @@ void CancelAfterInvoke6(CoreEnd2endTest& test,
                         std::unique_ptr<CancellationMode> mode,
                         Duration timeout) {
   test.InitClient(ChannelArgs());
+  // This is a workaround for the flakiness that if the server ever enters
+  // GracefulShutdown for whatever reason while the client has already been
+  // shutdown, the test would not timeout and fail.
   test.InitServer(ChannelArgs().Set(GRPC_ARG_PING_TIMEOUT_MS, 5000));
   auto c = test.NewClientCall("/service/method").Timeout(timeout).Create();
   CoreEnd2endTest::IncomingStatusOnClient server_status;
@@ -61,6 +64,9 @@ void CancelAfterInvoke5(CoreEnd2endTest& test,
                         std::unique_ptr<CancellationMode> mode,
                         Duration timeout) {
   test.InitClient(ChannelArgs());
+  // This is a workaround for the flakiness that if the server ever enters
+  // GracefulShutdown for whatever reason while the client has already been
+  // shutdown, the test would not timeout and fail.
   test.InitServer(ChannelArgs().Set(GRPC_ARG_PING_TIMEOUT_MS, 5000));
   auto c = test.NewClientCall("/service/method").Timeout(timeout).Create();
   CoreEnd2endTest::IncomingStatusOnClient server_status;
@@ -82,6 +88,9 @@ void CancelAfterInvoke4(CoreEnd2endTest& test,
                         std::unique_ptr<CancellationMode> mode,
                         Duration timeout) {
   test.InitClient(ChannelArgs());
+  // This is a workaround for the flakiness that if the server ever enters
+  // GracefulShutdown for whatever reason while the client has already been
+  // shutdown, the test would not timeout and fail.
   test.InitServer(ChannelArgs().Set(GRPC_ARG_PING_TIMEOUT_MS, 5000));
   auto c = test.NewClientCall("/service/method").Timeout(timeout).Create();
   CoreEnd2endTest::IncomingStatusOnClient server_status;
@@ -102,6 +111,9 @@ void CancelAfterInvoke3(CoreEnd2endTest& test,
                         std::unique_ptr<CancellationMode> mode,
                         Duration timeout) {
   test.InitClient(ChannelArgs());
+  // This is a workaround for the flakiness that if the server ever enters
+  // GracefulShutdown for whatever reason while the client has already been
+  // shutdown, the test would not timeout and fail.
   test.InitServer(ChannelArgs().Set(GRPC_ARG_PING_TIMEOUT_MS, 5000));
   auto c = test.NewClientCall("/service/method").Timeout(timeout).Create();
   CoreEnd2endTest::IncomingStatusOnClient server_status;

test/core/end2end/tests/cancel_with_status.cc

@@ -66,6 +66,9 @@ CORE_END2END_TEST(CoreEnd2endTest, CancelWithStatus2) {
 
 CORE_END2END_TEST(CoreEnd2endTest, CancelWithStatus3) {
   InitClient(ChannelArgs());
+  // This is a workaround for the flakiness that if the server ever enters
+  // GracefulShutdown for whatever reason while the client has already been
+  // shutdown, the test would not timeout and fail.
   InitServer(ChannelArgs().Set(GRPC_ARG_PING_TIMEOUT_MS, 5000));
   auto c = NewClientCall("/foo").Timeout(Duration::Minutes(1)).Create();
   CoreEnd2endTest::IncomingMetadata server_initial_metadata;
@@ -87,6 +90,9 @@ CORE_END2END_TEST(CoreEnd2endTest, CancelWithStatus3) {
 
 CORE_END2END_TEST(CoreEnd2endTest, CancelWithStatus4) {
   InitClient(ChannelArgs());
+  // This is a workaround for the flakiness that if the server ever enters
+  // GracefulShutdown for whatever reason while the client has already been
+  // shutdown, the test would not timeout and fail.
   InitServer(ChannelArgs().Set(GRPC_ARG_PING_TIMEOUT_MS, 5000));
   auto c = NewClientCall("/foo").Timeout(Duration::Minutes(1)).Create();
   CoreEnd2endTest::IncomingMetadata server_initial_metadata;

test/core/end2end/tests/retry_cancel_after_first_attempt_starts.cc

@@ -31,6 +31,9 @@ namespace {
 // Tests that we can unref a call after the first attempt starts but
 // before any ops complete.  This should not cause a memory leak.
 CORE_END2END_TEST(RetryTest, RetryCancelAfterFirstAttemptStarts) {
+  // This is a workaround for the flakiness that if the server ever enters
+  // GracefulShutdown for whatever reason while the client has already been
+  // shutdown, the test would not timeout and fail.
   InitServer(ChannelArgs().Set(GRPC_ARG_PING_TIMEOUT_MS, 4000));
   InitClient(ChannelArgs().Set(
       GRPC_ARG_SERVICE_CONFIG,

test/core/end2end/tests/retry_cancel_with_multiple_send_batches.cc

@@ -27,6 +27,7 @@
 #include <grpc/impl/channel_arg_names.h>
 #include <grpc/status.h>
 
+#include "src/core/ext/transport/chttp2/transport/internal.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack.h"
@@ -48,7 +49,10 @@ namespace {
 // Tests cancellation with multiple send op batches.
 void TestRetryCancelWithMultipleSendBatches(
     CoreEnd2endTest& test, std::unique_ptr<CancellationMode> mode) {
-  test.InitServer(ChannelArgs());
+  // This is a workaround for the flakiness that if the server ever enters
+  // GracefulShutdown for whatever reason while the client has already been
+  // shutdown, the test would not timeout and fail.
+  test.InitServer(ChannelArgs().Set(GRPC_ARG_PING_TIMEOUT_MS, 5000));
   test.InitClient(
       ChannelArgs()
           .Set(