From a7d9865961314fe5bb25734275c149a604ea5aa9 Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Thu, 23 Apr 2020 10:10:07 -0700 Subject: [PATCH] Fixing xds_end2endtest.cc BackendsRestart flake test Increase the number of RPCs after shutdown to ensure all backends are down before restarting. --- test/cpp/end2end/xds_end2end_test.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 9a4befb0388..9b6999639ac 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -1805,7 +1805,16 @@ TEST_P(BasicTest, BackendsRestart) { WaitForAllBackends(); // Stop backends. RPCs should fail. ShutdownAllBackends(); - CheckRpcSendFailure(); + // Sending multiple failed requests instead of just one to ensure that the + // client notices that all backends are down before we restart them. If we + // didn't do this, then a single RPC could fail here due to the race condition + // between the LB pick and the GOAWAY from the chosen backend being shut down, + // which would not actually prove that the client noticed that all of the + // backends are down. Then, when we send another request below (which we + // expect to succeed), if the callbacks happen in the wrong order, the same + // race condition could happen again due to the client not yet having noticed + // that the backends were all down. + CheckRpcSendFailure(num_backends_); // Restart all backends. RPCs should start succeeding again. StartAllBackends(); CheckRpcSendOk(1, RpcOptions().set_timeout_ms(2000).set_wait_for_ready(true));