[iOS/ObjC] Adding test flake repeat run support for interop (#30287)

src/objective-c/tests/BUILD

@@ -198,9 +198,15 @@ grpc_objc_ios_unit_test(
 )
 
 grpc_objc_ios_unit_test(
-    name = "InteropTestsLocal",
+    name = "InteropTestsLocalCleartext",
     deps = [
         ":InteropTestsLocalCleartext-lib",
+    ],
+)
+
+grpc_objc_ios_unit_test(
+    name = "InteropTestsLocalSSL",
+    deps = [
         ":InteropTestsLocalSSL-lib",
     ],
 )

src/objective-c/tests/Common/TestUtils.h

@@ -17,9 +17,21 @@
  */
 
 #import <Foundation/Foundation.h>
+#import <XCTest/XCTest.h>
 
 NS_ASSUME_NONNULL_BEGIN
 
+/* Default test timeout in seconds for interopt test. */
+FOUNDATION_EXPORT const NSTimeInterval GRPCInteropTestTimeoutDefault;
+
+// Block typedef for waiting for a target group of expectations via XCTWaiter.
+typedef void (^GRPCTestWaiter)(XCTestCase *testCase, NSArray<XCTestExpectation *> *expectations,
+                               NSTimeInterval timeout);
+
+// Block typedef for a test run. Test run should call waiter to wait for a group of expectations
+// with timeout.
+typedef void (^GRPCTestRunBlock)(GRPCTestWaiter waiterBlock);
+
 /**
  * Common utility to fetch plain text local interop server address.
  *
@@ -46,4 +58,17 @@ FOUNDATION_EXPORT NSString *GRPCGetRemoteInteropTestServerAddress(void);
  */
 FOUNDATION_EXPORT void GRPCPrintInteropTestServerDebugInfo(void);
 
+/**
+ * Common utility to run a test block until success, up to predefined number of repeats.
+ * @param testBlock Target test block to be invoked by the utility function. The block will be
+ * invoked synchronously before the function returns.
+ * @return YES if test run succeeded within the repeat limit. NO otherwise.
+ */
+FOUNDATION_EXPORT BOOL GRPCTestRunWithFlakeRepeats(GRPCTestRunBlock testBlock);
+
+/**
+ * Common utility to reset gRPC call's active connections.
+ */
+FOUNDATION_EXPORT void GRPCResetCallConnections(void);
+
 NS_ASSUME_NONNULL_END

src/objective-c/tests/Common/TestUtils.m

@@ -16,10 +16,21 @@
 
 #import "TestUtils.h"
 
+#import <XCTest/XCTest.h>
+
+#import <GRPCClient/GRPCCall+ChannelArg.h>
+#import <GRPCClient/GRPCCall+Tests.h>
+
 // Utility macro to stringize preprocessor defines
 #define NSStringize_helper(x) #x
 #define NSStringize(x) @NSStringize_helper(x)
 
+// Default test flake repeat counts
+static const NSUInteger kGRPCDefaultTestFlakeRepeats = 2;
+
+// Default interop local test timeout.
+const NSTimeInterval GRPCInteropTestTimeoutDefault = 3.0;
+
 NSString *GRPCGetLocalInteropTestServerAddressPlainText() {
   static NSString *address;
   static dispatch_once_t onceToken;
@@ -50,6 +61,26 @@ NSString *GRPCGetRemoteInteropTestServerAddress() {
   return address;
 }
 
+// Helper function to retrieve falke repeat from env variable settings.
+static NSUInteger GRPCGetTestFlakeRepeats() {
+  static NSUInteger repeats = kGRPCDefaultTestFlakeRepeats;
+  static dispatch_once_t onceToken;
+  dispatch_once(&onceToken, ^{
+    NSString *repeatStr = [NSProcessInfo processInfo].environment[@"FLAKE_TEST_REPEATS"];
+    if (repeatStr != nil) {
+      repeats = [repeatStr integerValue];
+    }
+  });
+  return repeats;
+}
+
+void GRPCResetCallConnections() {
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+  [GRPCCall closeOpenConnections];
+#pragma clang diagnostic pop
+}
+
 void GRPCPrintInteropTestServerDebugInfo() {
   NSLog(@"local interop env: %@  macro: %@",
         [NSProcessInfo processInfo].environment[@"HOST_PORT_LOCAL"], NSStringize(HOST_PORT_LOCAL));
@@ -60,3 +91,30 @@ void GRPCPrintInteropTestServerDebugInfo() {
         [NSProcessInfo processInfo].environment[@"HOST_PORT_REMOTE"],
         NSStringize(HOST_PORT_REMOTE));
 }
+
+BOOL GRPCTestRunWithFlakeRepeats(GRPCTestRunBlock testBlock) {
+  NSInteger repeats = GRPCGetTestFlakeRepeats();
+  NSInteger runs = 0;
+
+  while (runs < repeats) {
+    __block XCTWaiterResult result;
+    GRPCResetCallConnections();
+
+    testBlock(^(XCTestCase *testCase, NSArray<XCTestExpectation *> *expectations,
+                NSTimeInterval timeout) {
+      if (runs < repeats - 1) {
+        result = [XCTWaiter waitForExpectations:expectations timeout:timeout];
+      } else {
+        XCTWaiter *waiter = [[XCTWaiter alloc] initWithDelegate:testCase];
+        result = [waiter waitForExpectations:expectations timeout:timeout];
+      }
+    });
+
+    if (result == XCTWaiterResultCompleted) {
+      return YES;
+    }
+    runs += 1;
+  }
+
+  return NO;
+}

src/objective-c/tests/InteropTests/InteropTests.m

@@ -35,6 +35,7 @@
 #import "src/objective-c/tests/RemoteTestClient/Test.pbobjc.h"
 #import "src/objective-c/tests/RemoteTestClient/Test.pbrpc.h"
 
+#import "../Common/TestUtils.h"
 #import "InteropTestsBlockCallbacks.h"
 
 #define TEST_TIMEOUT 64
@@ -466,12 +467,7 @@ static dispatch_once_t initGlobalInterceptorFactory;
   self.continueAfterFailure = NO;
 
   [GRPCCall resetHostSettings];
-
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wdeprecated-declarations"
-  [GRPCCall closeOpenConnections];
-#pragma clang diagnostic pop
-
+  GRPCResetCallConnections();
   _service = [[self class] host] ? [RMTTestService serviceWithHost:[[self class] host]] : nil;
 }
 
@@ -480,27 +476,28 @@ static dispatch_once_t initGlobalInterceptorFactory;
 }
 
 - (void)testEmptyUnaryRPC {
-  XCTAssertNotNil([[self class] host]);
-  __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyUnary"];
+  GRPCTestRunWithFlakeRepeats(^(GRPCTestWaiter waiter) {
+    RMTTestService *service = [RMTTestService serviceWithHost:[[self class] host]];
+    __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyUnary"];
 
-  GPBEmpty *request = [GPBEmpty message];
-
-  __weak RMTTestService *weakService = _service;
-  [_service emptyCallWithRequest:request
-                         handler:^(GPBEmpty *response, NSError *error) {
-                           if (weakService == nil) {
-                             return;
-                           }
+    GPBEmpty *request = [GPBEmpty message];
 
-                           XCTAssertNil(error, @"Finished with unexpected error: %@", error);
+    __weak RMTTestService *weakService = service;
+    [service emptyCallWithRequest:request
+                          handler:^(GPBEmpty *response, NSError *error) {
+                            if (weakService == nil) {
+                              return;
+                            }
 
-                           id expectedResponse = [GPBEmpty message];
-                           XCTAssertEqualObjects(response, expectedResponse);
+                            XCTAssertNil(error, @"Finished with unexpected error: %@", error);
 
-                           [expectation fulfill];
-                         }];
+                            id expectedResponse = [GPBEmpty message];
+                            XCTAssertEqualObjects(response, expectedResponse);
 
-  [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil];
+                            [expectation fulfill];
+                          }];
+    waiter(self, @[ expectation ], GRPCInteropTestTimeoutDefault);
+  });
 }
 
 - (void)testEmptyUnaryRPCWithV2API {

tools/internal_ci/macos/grpc_objc_bazel_test.sh

@@ -47,7 +47,8 @@ EXAMPLE_TARGETS=(
 TEST_TARGETS=(
   # TODO(jtattermusch): ideally we'd say "//src/objective-c/tests/..." but not all the targets currently build
   # TODO(jtattermusch): make //src/objective-c/tests:TvTests test pass with bazel
-  //src/objective-c/tests:InteropTestsLocal
+  //src/objective-c/tests:InteropTestsLocalCleartext
+  //src/objective-c/tests:InteropTestsLocalSSL
   //src/objective-c/tests:InteropTestsRemote
   //src/objective-c/tests:MacTests
   //src/objective-c/tests:UnitTests
@@ -99,6 +100,7 @@ objc_bazel_tests/bazel_wrapper \
   $BAZEL_FLAGS \
   --test_env HOST_PORT_LOCAL=localhost:$PLAIN_PORT \
   --test_env HOST_PORT_LOCALSSL=localhost:$TLS_PORT \
+  --test_env FLAKE_TEST_REPEATS=3 \
   -- \
   "${EXAMPLE_TARGETS[@]}" \
   "${TEST_TARGETS[@]}"