Merge pull request #20336 from rmstar/perftest2

iOS performance test enhancements

src/objective-c/CronetFramework.podspec

@@ -30,7 +30,7 @@
 
 Pod::Spec.new do |s|
   s.name         = "CronetFramework"
-  v = '0.0.4'
+  v = '0.0.5'
   s.version      = v
   s.summary      = "Cronet, precompiled and used as a framework."
   s.homepage     = "http://chromium.org"

src/objective-c/tests/PerfTests/PerfTests.m

@@ -76,11 +76,6 @@ extern const char *kCFStreamVarName;
 
 @end
 
-BOOL isUsingCFStream() {
-  NSString *enabled = @(getenv(kCFStreamVarName));
-  return [enabled isEqualToString:@"1"];
-}
-
 #pragma mark Tests
 
 @implementation PerfTests {
@@ -118,12 +113,6 @@ BOOL isUsingCFStream() {
   return nil;
 }
 
-+ (void)setUp {
-  setenv("GRPC_TRACE", "tcp", 1);
-  setenv("GRPC_VERBOSITY", "DEBUG", 1);
-  NSLog(@"In setUp");
-}
-
 - (void)setUp {
   self.continueAfterFailure = NO;
 
@@ -137,6 +126,10 @@ BOOL isUsingCFStream() {
   _service = [[self class] host] ? [RMTTestService serviceWithHost:[[self class] host]] : nil;
 }
 
+- (BOOL)isUsingCFStream {
+  return [NSStringFromClass([self class]) isEqualToString:@"PerfTestsCFStreamSSL"];
+}
+
 - (void)pingPongV2APIWithRequest:(RMTStreamingOutputCallRequest *)request
                      numMessages:(int)numMessages
                          options:(GRPCMutableCallOptions *)options {
@@ -265,37 +258,41 @@ BOOL isUsingCFStream() {
   }];
 }
 
-- (void)unaryRPCWithRequest:(RMTSimpleRequest *)request
+- (void)unaryRPCsWithServices:(NSArray<RMTTestService *> *)services
-                numMessages:(int)numMessages
+                      request:(RMTSimpleRequest *)request
-                callOptions:(GRPCMutableCallOptions *)options {
+              callsPerService:(int)callsPerService
-  const int kOutstandingRPCs = 10;
+          maxOutstandingCalls:(int)maxOutstandingCalls
-  NSAssert(numMessages > kOutstandingRPCs, @"Number of RPCs must be > %d", kOutstandingRPCs);
+                  callOptions:(GRPCMutableCallOptions *)options {
   __weak XCTestExpectation *expectation = [self expectationWithDescription:@"unaryRPC"];
 
-  dispatch_semaphore_t sema = dispatch_semaphore_create(kOutstandingRPCs);
+  dispatch_semaphore_t sema = dispatch_semaphore_create(maxOutstandingCalls);
   __block int index = 0;
 
-  for (int i = 0; i < numMessages; ++i) {
+  for (RMTTestService *service in services) {
-    GRPCUnaryProtoCall *call = [_service
+    for (int i = 0; i < callsPerService; ++i) {
-        unaryCallWithMessage:request
+      GRPCUnaryProtoCall *call = [service
-             responseHandler:[[PerfTestsBlockCallbacks alloc]
+          unaryCallWithMessage:request
-                                 initWithInitialMetadataCallback:nil
+               responseHandler:[[PerfTestsBlockCallbacks alloc]
-                                                 messageCallback:nil
+                                   initWithInitialMetadataCallback:nil
-                                                   closeCallback:^(NSDictionary *trailingMetadata,
+                                                   messageCallback:nil
-                                                                   NSError *error) {
+                                                     closeCallback:^(NSDictionary *trailingMetadata,
-                                                     dispatch_semaphore_signal(sema);
+                                                                     NSError *error) {
-                                                     @synchronized(self) {
+                                                       dispatch_semaphore_signal(sema);
-                                                       ++index;
+                                                       @synchronized(self) {
-                                                       if (index == numMessages) {
+                                                         ++index;
-                                                         [expectation fulfill];
+                                                         if (index ==
+                                                             callsPerService * [services count]) {
+                                                           [expectation fulfill];
+                                                         }
                                                        }
-                                                     }
 
-                                                   }]
+                                                     }]
-                 callOptions:options];
+                   callOptions:options];
-
+      dispatch_time_t timeout =
-    dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER);
+          dispatch_time(DISPATCH_TIME_NOW, (int64_t)(TEST_TIMEOUT * NSEC_PER_SEC));
-    [call start];
+      dispatch_semaphore_wait(sema, timeout);
+      [call start];
+    }
   }
 
   [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil];
@@ -303,7 +300,7 @@ BOOL isUsingCFStream() {
 
 - (void)testUnaryRPC {
   // Workaround Apple CFStream bug
-  if (isUsingCFStream()) {
+  if ([self isUsingCFStream]) {
     return;
   }
 
@@ -317,10 +314,54 @@ BOOL isUsingCFStream() {
   options.hostNameOverride = [[self class] hostNameOverride];
 
   // warm up
-  [self unaryRPCWithRequest:request numMessages:50 callOptions:options];
+  [self unaryRPCsWithServices:@[ self->_service ]
+                      request:request
+              callsPerService:50
+          maxOutstandingCalls:10
+                  callOptions:options];
+
+  [self measureBlock:^{
+    [self unaryRPCsWithServices:@[ self->_service ]
+                        request:request
+                callsPerService:50
+            maxOutstandingCalls:10
+                    callOptions:options];
+  }];
+}
+
+- (void)testMultipleChannels {
+  NSString *port = [[[self class] host] componentsSeparatedByString:@":"][1];
+  int kNumAddrs = 10;
+  NSMutableArray<NSString *> *addrs = [NSMutableArray arrayWithCapacity:kNumAddrs];
+  NSMutableArray<RMTTestService *> *services = [NSMutableArray arrayWithCapacity:kNumAddrs];
+  for (int i = 0; i < kNumAddrs; ++i) {
+    addrs[i] = [NSString stringWithFormat:@"127.0.0.%d", (i + 1)];
+    NSString *hostWithPort = [NSString stringWithFormat:@"%@:%@", addrs[i], port];
+    services[i] = [RMTTestService serviceWithHost:hostWithPort];
+  }
+
+  RMTSimpleRequest *request = [RMTSimpleRequest message];
+  request.responseSize = 0;
+  request.payload.body = [NSMutableData dataWithLength:0];
+
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transport = [[self class] transport];
+  options.PEMRootCertificates = [[self class] PEMRootCertificates];
+  options.hostNameOverride = [[self class] hostNameOverride];
+
+  // warm up
+  [self unaryRPCsWithServices:services
+                      request:request
+              callsPerService:100
+          maxOutstandingCalls:100
+                  callOptions:options];
 
   [self measureBlock:^{
-    [self unaryRPCWithRequest:request numMessages:50 callOptions:options];
+    [self unaryRPCsWithServices:services
+                        request:request
+                callsPerService:100
+            maxOutstandingCalls:100
+                    callOptions:options];
   }];
 }
 

src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/PerfTests.xcscheme

@@ -41,6 +41,36 @@
                <Test
                   Identifier = "PerfTests">
                </Test>
+               <Test
+                  Identifier = "PerfTestsCFStreamCleartext">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsCronet/testPingPongRPCWithFlowControl">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsCronet/testPingPongRPCWithInterceptor">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsCronet/testPingPongRPCWithV1API">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsNoCFStreamCleartext">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsNoCFStreamSSL">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsNoCFStreamSSL/testPingPongRPCWithFlowControl">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsNoCFStreamSSL/testPingPongRPCWithInterceptor">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsNoCFStreamSSL/testPingPongRPCWithV1API">
+               </Test>
+               <Test
+                  Identifier = "TestBase">
+               </Test>
             </SkippedTests>
          </TestableReference>
       </Testables>

src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/PerfTestsPosix.xcscheme

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Scheme
+   LastUpgradeVersion = "1010"
+   version = "1.3">
+   <BuildAction
+      parallelizeBuildables = "YES"
+      buildImplicitDependencies = "YES">
+   </BuildAction>
+   <TestAction
+      buildConfiguration = "Test"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      shouldUseLaunchSchemeArgsEnv = "YES">
+      <Testables>
+         <TestableReference
+            skipped = "NO">
+            <BuildableReference
+               BuildableIdentifier = "primary"
+               BlueprintIdentifier = "B0F2D0B9232991BA008C2575"
+               BuildableName = "PerfTests.xctest"
+               BlueprintName = "PerfTests"
+               ReferencedContainer = "container:Tests.xcodeproj">
+            </BuildableReference>
+            <SkippedTests>
+               <Test
+                  Identifier = "PerfTests">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsCFStreamCleartext">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsCFStreamSSL">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsCronet">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsNoCFStreamCleartext">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsNoCFStreamSSL/testPingPongRPCWithFlowControl">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsNoCFStreamSSL/testPingPongRPCWithInterceptor">
+               </Test>
+               <Test
+                  Identifier = "PerfTestsNoCFStreamSSL/testPingPongRPCWithV1API">
+               </Test>
+               <Test
+                  Identifier = "TestBase">
+               </Test>
+            </SkippedTests>
+         </TestableReference>
+      </Testables>
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </TestAction>
+   <LaunchAction
+      buildConfiguration = "Debug"
+      selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
+      selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
+      launchStyle = "0"
+      useCustomWorkingDirectory = "NO"
+      ignoresPersistentStateOnLaunch = "NO"
+      debugDocumentVersioning = "YES"
+      debugServiceExtension = "internal"
+      allowLocationSimulation = "YES">
+      <AdditionalOptions>
+      </AdditionalOptions>
+   </LaunchAction>
+   <ProfileAction
+      buildConfiguration = "Release"
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      savedToolIdentifier = ""
+      useCustomWorkingDirectory = "NO"
+      debugDocumentVersioning = "YES">
+   </ProfileAction>
+   <AnalyzeAction
+      buildConfiguration = "Debug">
+   </AnalyzeAction>
+   <ArchiveAction
+      buildConfiguration = "Release"
+      revealArchiveInOrganizer = "YES">
+   </ArchiveAction>
+</Scheme>

src/objective-c/tests/run_one_test.sh

@@ -43,7 +43,25 @@ TLS_PORT=$(curl localhost:32766/get)
 $INTEROP --port=$PLAIN_PORT --max_send_message_size=8388608 &
 $INTEROP --port=$TLS_PORT --max_send_message_size=8388608 --use_tls &
 
-trap 'kill -9 `jobs -p` ; echo "EXIT TIME:  $(date)"' EXIT
+# Create loopback aliases for iOS performance tests
+if [ $SCHEME == PerfTests ] || [ $SCHEME == PerfTestsPosix ]; then
+for ((i=2;i<11;i++))
+do
+    sudo ifconfig lo0 alias 127.0.0.$i up
+done
+fi
+
+function finish {
+    if [ $SCHEME == PerfTests ] || [ $SCHEME == PerfTestsPosix ]; then
+    for ((i=2;i<11;i++))
+    do
+        sudo ifconfig lo0 -alias 127.0.0.$i
+    done
+    fi
+    kill -9 `jobs -p`
+    echo "EXIT TIME:  $(date)"
+}
+trap finish EXIT
 
 set -o pipefail
 
@@ -59,6 +77,7 @@ elif [ $PLATFORM == tvos ]; then
 DESTINATION='platform=tvOS Simulator,name=Apple TV'
 fi
 
+
 xcodebuild \
     -workspace Tests.xcworkspace \
     -scheme $SCHEME \
@@ -70,3 +89,4 @@ xcodebuild \
     | egrep -v "$XCODEBUILD_FILTER" \
     | egrep -v '^$' \
     | egrep -v "(GPBDictionary|GPBArray)" -
+

tools/run_tests/run_tests.py

@@ -1187,6 +1187,24 @@ class ObjCLanguage(object):
                 environ={
                     'SCHEME': 'CronetTests'
                 }))
+        out.append(
+            self.config.job_spec(
+                ['src/objective-c/tests/run_one_test.sh'],
+                timeout_seconds=30 * 60,
+                shortname='ios-perf-test',
+                cpu_cost=1e6,
+                environ={
+                    'SCHEME': 'PerfTests'
+                }))
+        out.append(
+            self.config.job_spec(
+                ['src/objective-c/tests/run_one_test.sh'],
+                timeout_seconds=30 * 60,
+                shortname='ios-perf-test-posix',
+                cpu_cost=1e6,
+                environ={
+                    'SCHEME': 'PerfTestsPosix'
+                }))
         out.append(
             self.config.job_spec(
                 ['test/cpp/ios/build_and_run_tests.sh'],