Merge pull request #4449 from jtattermusch/csharp_test_restructuring

C# test restructuring

src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj

@@ -91,6 +91,7 @@
     <Compile Include="ContextPropagationTest.cs" />
     <Compile Include="MetadataTest.cs" />
     <Compile Include="PerformanceTest.cs" />
+    <Compile Include="SanityTest.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
   <ItemGroup>

src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs

@@ -167,18 +167,18 @@ namespace Grpc.Core.Internal.Tests
                 () => Timespec.FromDateTime(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified)));
         }
             
-        // Test attribute commented out to prevent running as part of the default test suite.
+        [Test]
-        // [Test]
+        [Category("Performance")]
-        // [Category("Performance")]
+        [Ignore("Prevent running on Jenkins")]
         public void NowBenchmark() 
         {
             // approx Timespec.Now latency <33ns
             BenchmarkUtil.RunBenchmark(10000000, 1000000000, () => { var now = Timespec.Now; });
         }
-
+            
-        // Test attribute commented out to prevent running as part of the default test suite.
+        [Test]
-        // [Test]
+        [Category("Performance")]
-        // [Category("Performance")]
+        [Ignore("Prevent running on Jenkins")]
         public void PreciseNowBenchmark()
         {
             // approx Timespec.PreciseNow latency <18ns (when compiled with GRPC_TIMERS_RDTSC)

src/csharp/Grpc.Core.Tests/PInvokeTest.cs

@@ -59,6 +59,8 @@ namespace Grpc.Core.Tests
         [Test]
         public void CompletionQueueCreateDestroyBenchmark()
         {
+            GrpcEnvironment.AddRef();  // completion queue requires gRPC environment being initialized.
+
             BenchmarkUtil.RunBenchmark(
                 10, 10,
                 () =>
@@ -66,6 +68,8 @@ namespace Grpc.Core.Tests
                     CompletionQueueSafeHandle cq = CompletionQueueSafeHandle.Create();
                     cq.Dispose();
                 });
+
+            GrpcEnvironment.Release();
         }
 
         /// <summary>

src/csharp/Grpc.Core.Tests/PerformanceTest.cs

@@ -67,10 +67,10 @@ namespace Grpc.Core.Tests
             channel.ShutdownAsync().Wait();
             server.ShutdownAsync().Wait();
         }
-
+            
-        // Test attribute commented out to prevent running as part of the default test suite.
+        [Test]
-        //[Test]
+        [Category("Performance")]
-        //[Category("Performance")]
+        [Ignore("Prevent running on Jenkins")]
         public void UnaryCallPerformance()
         {
             var profiler = new BasicProfiler();

src/csharp/Grpc.Core.Tests/SanityTest.cs

@@ -0,0 +1,125 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using Grpc.Core;
+using Grpc.Core.Internal;
+using Grpc.Core.Utils;
+using NUnit.Framework;
+
+namespace Grpc.Core.Tests
+{
+    public class SanityTest
+    {
+        /// <summary>
+        /// Because we depend on a native library, sometimes when things go wrong, the
+        /// entire NUnit test process crashes. To be able to track down problems better,
+        /// the NUnit tests are run by run_tests.py script in a separate process per test class.
+        /// The list of tests to run is stored in src/csharp/tests.json.
+        /// This test checks that the tests.json file is up to date by discovering all the
+        /// existing NUnit tests in all test assemblies and comparing to contents of tests.json.
+        /// </summary>
+        [Test]
+        public void TestsJsonUpToDate()
+        {
+            var testClasses = DiscoverAllTestClasses();
+            string testsJson = GetTestsJson();
+
+            // we don't have a JSON parser at hand, but check that the test class
+            // name is contained in the file instead.
+            foreach (var className in testClasses) {
+                Assert.IsTrue(testsJson.Contains(className),
+                    string.Format("Test class \"{0}\" is missing in C# tests.json file", className));
+            }
+        }
+
+        /// <summary>
+        /// Gets list of all test classes obtained by inspecting all the test assemblies.
+        /// </summary>
+        private List<string> DiscoverAllTestClasses()
+        {
+            var assemblies = GetTestAssemblies();
+
+            var testClasses = new List<string>();
+            foreach (var assembly in assemblies)
+            {
+                foreach (var t in assembly.GetTypes())
+                {
+                    foreach (var m in t.GetMethods())
+                    {
+                        var attributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true);
+                        if (attributes.Length > 0)
+                        {
+                            testClasses.Add(t.FullName);
+                            break;
+                        }
+
+                    }
+                }
+            }
+            testClasses.Sort();
+            return testClasses;
+        }
+
+        private string GetTestsJson()
+        {
+            var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+            var testsJsonFile = Path.Combine(assemblyDir, "..", "..", "..", "tests.json");
+
+            return File.ReadAllText(testsJsonFile);
+        }
+
+        private List<Assembly> GetTestAssemblies()
+        {
+            var result = new List<Assembly>();
+            var executingAssembly = Assembly.GetExecutingAssembly();
+
+            result.Add(executingAssembly);
+
+            var otherAssemblies = new[] {
+                "Grpc.Examples.Tests",
+                "Grpc.HealthCheck.Tests",
+                "Grpc.IntegrationTesting"
+            };
+            foreach (var assemblyName in otherAssemblies)
+            {
+                var location = executingAssembly.Location.Replace("Grpc.Core.Tests", assemblyName);
+                result.Add(Assembly.LoadFrom(location));
+            }
+            return result;
+        }
+    }
+}

src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs

@@ -140,12 +140,14 @@ namespace Grpc.IntegrationTesting
         }
 
         [Test]
+        [Ignore("TODO: see #4427")]
         public async Task StatusCodeAndMessage()
         {
             await InteropClient.RunStatusCodeAndMessageAsync(client);
         }
 
         [Test]
+        [Ignore("TODO: see #4427")]
         public void UnimplementedMethod()
         {
             InteropClient.RunUnimplementedMethod(UnimplementedService.NewClient(channel));

src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs

@@ -75,9 +75,10 @@ namespace Grpc.IntegrationTesting
             serverRunner.StopAsync().Wait();
         }
 
-        // Test attribute commented out to prevent running as part of the default test suite.
+
-        //[Test]
+        [Test]
-        //[Category("Performance")]
+        [Category("Performance")]
+        [Ignore("Prevent running on Jenkins")]
         public async Task ClientServerRunner()
         {
             var config = new ClientConfig

src/csharp/tests.json

@@ -0,0 +1,45 @@
+{
+  "assemblies": [
+    "Grpc.Core.Tests",
+    "Grpc.Examples.Tests",
+    "Grpc.HealthCheck.Tests",
+    "Grpc.IntegrationTesting"
+  ],
+  "tests": [
+    "Grpc.Core.Internal.Tests.AsyncCallTest",
+    "Grpc.Core.Internal.Tests.ChannelArgsSafeHandleTest",
+    "Grpc.Core.Internal.Tests.CompletionQueueEventTest",
+    "Grpc.Core.Internal.Tests.CompletionQueueSafeHandleTest",
+    "Grpc.Core.Internal.Tests.MetadataArraySafeHandleTest",
+    "Grpc.Core.Internal.Tests.TimespecTest",
+    "Grpc.Core.Tests.CallCredentialsTest",
+    "Grpc.Core.Tests.CallOptionsTest",
+    "Grpc.Core.Tests.ChannelCredentialsTest",
+    "Grpc.Core.Tests.ChannelOptionsTest",
+    "Grpc.Core.Tests.ChannelTest",
+    "Grpc.Core.Tests.ClientServerTest",
+    "Grpc.Core.Tests.CompressionTest",
+    "Grpc.Core.Tests.ContextPropagationTest",
+    "Grpc.Core.Tests.GrpcEnvironmentTest",
+    "Grpc.Core.Tests.MarshallingErrorsTest",
+    "Grpc.Core.Tests.MetadataTest",
+    "Grpc.Core.Tests.NUnitVersionTest",
+    "Grpc.Core.Tests.PerformanceTest",
+    "Grpc.Core.Tests.PInvokeTest",
+    "Grpc.Core.Tests.ResponseHeadersTest",
+    "Grpc.Core.Tests.SanityTest",
+    "Grpc.Core.Tests.ServerTest",
+    "Grpc.Core.Tests.ShutdownTest",
+    "Grpc.Core.Tests.TimeoutsTest",
+    "Grpc.Core.Tests.UserAgentStringTest",
+    "Math.Tests.MathClientServerTest",
+    "Grpc.HealthCheck.Tests.HealthClientServerTest",
+    "Grpc.HealthCheck.Tests.HealthServiceImplTest",
+    "Grpc.IntegrationTesting.HeaderInterceptorTest",
+    "Grpc.IntegrationTesting.HistogramTest",
+    "Grpc.IntegrationTesting.InteropClientServerTest",
+    "Grpc.IntegrationTesting.MetadataCredentialsTest",
+    "Grpc.IntegrationTesting.RunnerClientServerTest",
+    "Grpc.IntegrationTesting.SslCredentialsTest"
+  ]
+}

tools/run_tests/build_csharp.sh

@@ -30,7 +30,7 @@
 
 set -ex
 
-if [ "$CONFIG" = "dbg" ] || [ "$CONFIG" = "gcov" ]
+if [ "$CONFIG" = "dbg" ]
 then
   MSBUILD_CONFIG="Debug"
 else

tools/run_tests/run_csharp.bat

@@ -6,16 +6,11 @@ setlocal
 cd /d %~dp0\..\..\src\csharp
 
 if not "%CONFIG%" == "gcov" (
-  @rem Run tests for assembly passed as 1st arg.
+  packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe %* || goto :error
-
-  @rem set UUID variable to a random GUID, we will use it to put TestResults.xml to a dedicated directory, so that parallel test runs don't collide
-  for /F %%i in ('powershell -Command "[guid]::NewGuid().ToString()"') do (set UUID=%%i)
-
-  packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe /domain:None -labels "%1/bin/Debug/%1.dll" -work test-results/%UUID% || goto :error
 ) else (
   @rem Run all tests with code coverage
 
-  packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -target:"packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe" -targetdir:"." -targetargs:"/domain:None -labels Grpc.Core.Tests/bin/Debug/Grpc.Core.Tests.dll Grpc.IntegrationTesting/bin/Debug/Grpc.IntegrationTesting.dll Grpc.Examples.Tests/bin/Debug/Grpc.Examples.Tests.dll Grpc.HealthCheck.Tests/bin/Debug/Grpc.HealthCheck.Tests.dll" -filter:"+[Grpc.Core]*"  -register:user -output:coverage_results.xml || goto :error
+  packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -target:"packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe" -targetdir:"." -targetargs:"%*" -filter:"+[Grpc.Core]*"  -register:user -output:coverage_results.xml || goto :error
 
   packages\ReportGenerator.2.3.2.0\tools\ReportGenerator.exe -reports:"coverage_results.xml" -targetdir:"..\..\reports\csharp_coverage" -reporttypes:"Html;TextSummary" || goto :error
 

tools/run_tests/run_csharp.sh

@@ -31,38 +31,25 @@
 set -ex
 
 CONFIG=${CONFIG:-opt}
-
 NUNIT_CONSOLE="mono packages/NUnit.Runners.2.6.4/tools/nunit-console.exe"
 
-if [ "$CONFIG" = "dbg" ] || [ "$CONFIG" = "gcov" ]
-then
-  MSBUILD_CONFIG="Debug"
-else
-  MSBUILD_CONFIG="Release"
-fi
-
 # change to gRPC repo root
 cd $(dirname $0)/../..
 
-root=`pwd`
+# path needs to be absolute
-export LD_LIBRARY_PATH=$root/libs/$CONFIG
+export LD_LIBRARY_PATH=$(pwd)/libs/$CONFIG
+
+(cd src/csharp; $NUNIT_CONSOLE $@)
 
 if [ "$CONFIG" = "gcov" ]
 then
-  (cd src/csharp; $NUNIT_CONSOLE -labels \
+  # Generate the csharp extension coverage report
-      "Grpc.Core.Tests/bin/$MSBUILD_CONFIG/Grpc.Core.Tests.dll" \
-      "Grpc.Examples.Tests/bin/$MSBUILD_CONFIG/Grpc.Examples.Tests.dll" \
-      "Grpc.HealthCheck.Tests/bin/$MSBUILD_CONFIG/Grpc.HealthCheck.Tests.dll" \
-      "Grpc.IntegrationTesting/bin/$MSBUILD_CONFIG/Grpc.IntegrationTesting.dll")
-
   gcov objs/gcov/src/csharp/ext/*.o
   lcov --base-directory . --directory . -c -o coverage.info
   lcov -e coverage.info '**/src/csharp/ext/*' -o coverage.info
   genhtml -o reports/csharp_ext_coverage --num-spaces 2 \
     -t 'gRPC C# native extension test coverage' coverage.info \
     --rc genhtml_hi_limit=95 --rc genhtml_med_limit=80 --no-prefix
-else
-  (cd src/csharp; $NUNIT_CONSOLE -labels "$1/bin/$MSBUILD_CONFIG/$1.dll")
 fi
 
 

tools/run_tests/run_tests.py

@@ -47,6 +47,7 @@ import tempfile
 import traceback
 import time
 import urllib2
+import uuid
 
 import jobset
 import report_utils
@@ -336,26 +337,42 @@ class CSharpLanguage(object):
     self.platform = platform_string()
 
   def test_specs(self, config, args):
-    assemblies = ['Grpc.Core.Tests',
+    with open('src/csharp/tests.json') as f:
-                  'Grpc.Examples.Tests',
+      tests_json = json.load(f)
-                  'Grpc.HealthCheck.Tests',
+    assemblies = tests_json['assemblies']
-                  'Grpc.IntegrationTesting']
+    tests = tests_json['tests']
+
+    msbuild_config = _WINDOWS_CONFIG[config.build_config]
+    assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a)
+                      for a in assemblies]
+
+    extra_args = ['-labels'] + assembly_files
+
     if self.platform == 'windows':
-      cmd = 'tools\\run_tests\\run_csharp.bat'
+      script_name = 'tools\\run_tests\\run_csharp.bat'
+      extra_args += ['-domain=None']
     else:
-      cmd = 'tools/run_tests/run_csharp.sh'
+      script_name = 'tools/run_tests/run_csharp.sh'
 
     if config.build_config == 'gcov':
       # On Windows, we only collect C# code coverage.
       # On Linux, we only collect coverage for native extension.
       # For code coverage all tests need to run as one suite.
-      return [config.job_spec([cmd], None,
+      return [config.job_spec([script_name] + extra_args, None,
-              environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+                              shortname='csharp.coverage',
+                              environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
     else:
-      return [config.job_spec([cmd, assembly],
+      specs = []
-              None, shortname=assembly,
+      for test in tests:
-              environ=_FORCE_ENVIRON_FOR_WRAPPERS)
+        cmdline = [script_name, '-run=%s' % test] + extra_args
-              for assembly in assemblies]
+        if self.platform == 'windows':
+          # use different output directory for each test to prevent
+          # TestResult.xml clash between parallel test runs.
+          cmdline += ['-work=test-result/%s' % uuid.uuid4()]
+        specs.append(config.job_spec(cmdline, None,
+                                     shortname='csharp.%s' % test,
+                                     environ=_FORCE_ENVIRON_FOR_WRAPPERS))
+      return specs
 
   def pre_build_steps(self):
     if self.platform == 'windows':
@@ -509,6 +526,7 @@ _LANGUAGES = {
 _WINDOWS_CONFIG = {
     'dbg': 'Debug',
     'opt': 'Release',
+    'gcov': 'Release',
     }