From 67be31c38c8df0eecbcfd8e5dca84a6bd3f049a3 Mon Sep 17 00:00:00 2001 From: "Denny C. Dai" Date: Thu, 14 Jul 2022 13:34:33 -0700 Subject: [PATCH] [iOS/ObjC] Adding test flake repeat run support for interop (#30287) --- src/objective-c/tests/BUILD | 8 ++- src/objective-c/tests/Common/TestUtils.h | 25 ++++++++ src/objective-c/tests/Common/TestUtils.m | 58 +++++++++++++++++++ .../tests/InteropTests/InteropTests.m | 41 ++++++------- .../internal_ci/macos/grpc_objc_bazel_test.sh | 4 +- 5 files changed, 112 insertions(+), 24 deletions(-) diff --git a/src/objective-c/tests/BUILD b/src/objective-c/tests/BUILD index d1fd748256f..42a6a4c048c 100644 --- a/src/objective-c/tests/BUILD +++ b/src/objective-c/tests/BUILD @@ -198,9 +198,15 @@ grpc_objc_ios_unit_test( ) grpc_objc_ios_unit_test( - name = "InteropTestsLocal", + name = "InteropTestsLocalCleartext", deps = [ ":InteropTestsLocalCleartext-lib", + ], +) + +grpc_objc_ios_unit_test( + name = "InteropTestsLocalSSL", + deps = [ ":InteropTestsLocalSSL-lib", ], ) diff --git a/src/objective-c/tests/Common/TestUtils.h b/src/objective-c/tests/Common/TestUtils.h index 98ccf7ba206..a12c87a591f 100644 --- a/src/objective-c/tests/Common/TestUtils.h +++ b/src/objective-c/tests/Common/TestUtils.h @@ -17,9 +17,21 @@ */ #import +#import NS_ASSUME_NONNULL_BEGIN +/* Default test timeout in seconds for interopt test. */ +FOUNDATION_EXPORT const NSTimeInterval GRPCInteropTestTimeoutDefault; + +// Block typedef for waiting for a target group of expectations via XCTWaiter. +typedef void (^GRPCTestWaiter)(XCTestCase *testCase, NSArray *expectations, + NSTimeInterval timeout); + +// Block typedef for a test run. Test run should call waiter to wait for a group of expectations +// with timeout. +typedef void (^GRPCTestRunBlock)(GRPCTestWaiter waiterBlock); + /** * Common utility to fetch plain text local interop server address. * @@ -46,4 +58,17 @@ FOUNDATION_EXPORT NSString *GRPCGetRemoteInteropTestServerAddress(void); */ FOUNDATION_EXPORT void GRPCPrintInteropTestServerDebugInfo(void); +/** + * Common utility to run a test block until success, up to predefined number of repeats. + * @param testBlock Target test block to be invoked by the utility function. The block will be + * invoked synchronously before the function returns. + * @return YES if test run succeeded within the repeat limit. NO otherwise. + */ +FOUNDATION_EXPORT BOOL GRPCTestRunWithFlakeRepeats(GRPCTestRunBlock testBlock); + +/** + * Common utility to reset gRPC call's active connections. + */ +FOUNDATION_EXPORT void GRPCResetCallConnections(void); + NS_ASSUME_NONNULL_END diff --git a/src/objective-c/tests/Common/TestUtils.m b/src/objective-c/tests/Common/TestUtils.m index 38fbaf9582f..50f40cabd2a 100644 --- a/src/objective-c/tests/Common/TestUtils.m +++ b/src/objective-c/tests/Common/TestUtils.m @@ -16,10 +16,21 @@ #import "TestUtils.h" +#import + +#import +#import + // Utility macro to stringize preprocessor defines #define NSStringize_helper(x) #x #define NSStringize(x) @NSStringize_helper(x) +// Default test flake repeat counts +static const NSUInteger kGRPCDefaultTestFlakeRepeats = 2; + +// Default interop local test timeout. +const NSTimeInterval GRPCInteropTestTimeoutDefault = 3.0; + NSString *GRPCGetLocalInteropTestServerAddressPlainText() { static NSString *address; static dispatch_once_t onceToken; @@ -50,6 +61,26 @@ NSString *GRPCGetRemoteInteropTestServerAddress() { return address; } +// Helper function to retrieve falke repeat from env variable settings. +static NSUInteger GRPCGetTestFlakeRepeats() { + static NSUInteger repeats = kGRPCDefaultTestFlakeRepeats; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + NSString *repeatStr = [NSProcessInfo processInfo].environment[@"FLAKE_TEST_REPEATS"]; + if (repeatStr != nil) { + repeats = [repeatStr integerValue]; + } + }); + return repeats; +} + +void GRPCResetCallConnections() { +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" + [GRPCCall closeOpenConnections]; +#pragma clang diagnostic pop +} + void GRPCPrintInteropTestServerDebugInfo() { NSLog(@"local interop env: %@ macro: %@", [NSProcessInfo processInfo].environment[@"HOST_PORT_LOCAL"], NSStringize(HOST_PORT_LOCAL)); @@ -60,3 +91,30 @@ void GRPCPrintInteropTestServerDebugInfo() { [NSProcessInfo processInfo].environment[@"HOST_PORT_REMOTE"], NSStringize(HOST_PORT_REMOTE)); } + +BOOL GRPCTestRunWithFlakeRepeats(GRPCTestRunBlock testBlock) { + NSInteger repeats = GRPCGetTestFlakeRepeats(); + NSInteger runs = 0; + + while (runs < repeats) { + __block XCTWaiterResult result; + GRPCResetCallConnections(); + + testBlock(^(XCTestCase *testCase, NSArray *expectations, + NSTimeInterval timeout) { + if (runs < repeats - 1) { + result = [XCTWaiter waitForExpectations:expectations timeout:timeout]; + } else { + XCTWaiter *waiter = [[XCTWaiter alloc] initWithDelegate:testCase]; + result = [waiter waitForExpectations:expectations timeout:timeout]; + } + }); + + if (result == XCTWaiterResultCompleted) { + return YES; + } + runs += 1; + } + + return NO; +} diff --git a/src/objective-c/tests/InteropTests/InteropTests.m b/src/objective-c/tests/InteropTests/InteropTests.m index d7a0098ca5b..40a2b2deaa9 100644 --- a/src/objective-c/tests/InteropTests/InteropTests.m +++ b/src/objective-c/tests/InteropTests/InteropTests.m @@ -35,6 +35,7 @@ #import "src/objective-c/tests/RemoteTestClient/Test.pbobjc.h" #import "src/objective-c/tests/RemoteTestClient/Test.pbrpc.h" +#import "../Common/TestUtils.h" #import "InteropTestsBlockCallbacks.h" #define TEST_TIMEOUT 64 @@ -466,12 +467,7 @@ static dispatch_once_t initGlobalInterceptorFactory; self.continueAfterFailure = NO; [GRPCCall resetHostSettings]; - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wdeprecated-declarations" - [GRPCCall closeOpenConnections]; -#pragma clang diagnostic pop - + GRPCResetCallConnections(); _service = [[self class] host] ? [RMTTestService serviceWithHost:[[self class] host]] : nil; } @@ -480,27 +476,28 @@ static dispatch_once_t initGlobalInterceptorFactory; } - (void)testEmptyUnaryRPC { - XCTAssertNotNil([[self class] host]); - __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyUnary"]; + GRPCTestRunWithFlakeRepeats(^(GRPCTestWaiter waiter) { + RMTTestService *service = [RMTTestService serviceWithHost:[[self class] host]]; + __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyUnary"]; - GPBEmpty *request = [GPBEmpty message]; - - __weak RMTTestService *weakService = _service; - [_service emptyCallWithRequest:request - handler:^(GPBEmpty *response, NSError *error) { - if (weakService == nil) { - return; - } + GPBEmpty *request = [GPBEmpty message]; - XCTAssertNil(error, @"Finished with unexpected error: %@", error); + __weak RMTTestService *weakService = service; + [service emptyCallWithRequest:request + handler:^(GPBEmpty *response, NSError *error) { + if (weakService == nil) { + return; + } - id expectedResponse = [GPBEmpty message]; - XCTAssertEqualObjects(response, expectedResponse); + XCTAssertNil(error, @"Finished with unexpected error: %@", error); - [expectation fulfill]; - }]; + id expectedResponse = [GPBEmpty message]; + XCTAssertEqualObjects(response, expectedResponse); - [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; + [expectation fulfill]; + }]; + waiter(self, @[ expectation ], GRPCInteropTestTimeoutDefault); + }); } - (void)testEmptyUnaryRPCWithV2API { diff --git a/tools/internal_ci/macos/grpc_objc_bazel_test.sh b/tools/internal_ci/macos/grpc_objc_bazel_test.sh index d4e95f92901..6854e35ef86 100755 --- a/tools/internal_ci/macos/grpc_objc_bazel_test.sh +++ b/tools/internal_ci/macos/grpc_objc_bazel_test.sh @@ -47,7 +47,8 @@ EXAMPLE_TARGETS=( TEST_TARGETS=( # TODO(jtattermusch): ideally we'd say "//src/objective-c/tests/..." but not all the targets currently build # TODO(jtattermusch): make //src/objective-c/tests:TvTests test pass with bazel - //src/objective-c/tests:InteropTestsLocal + //src/objective-c/tests:InteropTestsLocalCleartext + //src/objective-c/tests:InteropTestsLocalSSL //src/objective-c/tests:InteropTestsRemote //src/objective-c/tests:MacTests //src/objective-c/tests:UnitTests @@ -99,6 +100,7 @@ objc_bazel_tests/bazel_wrapper \ $BAZEL_FLAGS \ --test_env HOST_PORT_LOCAL=localhost:$PLAIN_PORT \ --test_env HOST_PORT_LOCALSSL=localhost:$TLS_PORT \ + --test_env FLAKE_TEST_REPEATS=3 \ -- \ "${EXAMPLE_TARGETS[@]}" \ "${TEST_TARGETS[@]}"