diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index a171855ee04..475c7923476 100644 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -91,6 +91,7 @@ + diff --git a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs index 48b5d78ca97..74f7f2497a9 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs @@ -167,18 +167,18 @@ namespace Grpc.Core.Internal.Tests () => Timespec.FromDateTime(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified))); } - // Test attribute commented out to prevent running as part of the default test suite. - // [Test] - // [Category("Performance")] + [Test] + [Category("Performance")] + [Ignore("Prevent running on Jenkins")] public void NowBenchmark() { // approx Timespec.Now latency <33ns BenchmarkUtil.RunBenchmark(10000000, 1000000000, () => { var now = Timespec.Now; }); } - - // Test attribute commented out to prevent running as part of the default test suite. - // [Test] - // [Category("Performance")] + + [Test] + [Category("Performance")] + [Ignore("Prevent running on Jenkins")] public void PreciseNowBenchmark() { // approx Timespec.PreciseNow latency <18ns (when compiled with GRPC_TIMERS_RDTSC) diff --git a/src/csharp/Grpc.Core.Tests/PInvokeTest.cs b/src/csharp/Grpc.Core.Tests/PInvokeTest.cs index 073c502daf3..af55cb0566b 100644 --- a/src/csharp/Grpc.Core.Tests/PInvokeTest.cs +++ b/src/csharp/Grpc.Core.Tests/PInvokeTest.cs @@ -59,6 +59,8 @@ namespace Grpc.Core.Tests [Test] public void CompletionQueueCreateDestroyBenchmark() { + GrpcEnvironment.AddRef(); // completion queue requires gRPC environment being initialized. + BenchmarkUtil.RunBenchmark( 10, 10, () => @@ -66,6 +68,8 @@ namespace Grpc.Core.Tests CompletionQueueSafeHandle cq = CompletionQueueSafeHandle.Create(); cq.Dispose(); }); + + GrpcEnvironment.Release(); } /// diff --git a/src/csharp/Grpc.Core.Tests/PerformanceTest.cs b/src/csharp/Grpc.Core.Tests/PerformanceTest.cs index 5516cd33774..68158399921 100644 --- a/src/csharp/Grpc.Core.Tests/PerformanceTest.cs +++ b/src/csharp/Grpc.Core.Tests/PerformanceTest.cs @@ -67,10 +67,10 @@ namespace Grpc.Core.Tests channel.ShutdownAsync().Wait(); server.ShutdownAsync().Wait(); } - - // Test attribute commented out to prevent running as part of the default test suite. - //[Test] - //[Category("Performance")] + + [Test] + [Category("Performance")] + [Ignore("Prevent running on Jenkins")] public void UnaryCallPerformance() { var profiler = new BasicProfiler(); diff --git a/src/csharp/Grpc.Core.Tests/SanityTest.cs b/src/csharp/Grpc.Core.Tests/SanityTest.cs new file mode 100644 index 00000000000..343ab1e85a1 --- /dev/null +++ b/src/csharp/Grpc.Core.Tests/SanityTest.cs @@ -0,0 +1,125 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using Grpc.Core; +using Grpc.Core.Internal; +using Grpc.Core.Utils; +using NUnit.Framework; + +namespace Grpc.Core.Tests +{ + public class SanityTest + { + /// + /// Because we depend on a native library, sometimes when things go wrong, the + /// entire NUnit test process crashes. To be able to track down problems better, + /// the NUnit tests are run by run_tests.py script in a separate process per test class. + /// The list of tests to run is stored in src/csharp/tests.json. + /// This test checks that the tests.json file is up to date by discovering all the + /// existing NUnit tests in all test assemblies and comparing to contents of tests.json. + /// + [Test] + public void TestsJsonUpToDate() + { + var testClasses = DiscoverAllTestClasses(); + string testsJson = GetTestsJson(); + + // we don't have a JSON parser at hand, but check that the test class + // name is contained in the file instead. + foreach (var className in testClasses) { + Assert.IsTrue(testsJson.Contains(className), + string.Format("Test class \"{0}\" is missing in C# tests.json file", className)); + } + } + + /// + /// Gets list of all test classes obtained by inspecting all the test assemblies. + /// + private List DiscoverAllTestClasses() + { + var assemblies = GetTestAssemblies(); + + var testClasses = new List(); + foreach (var assembly in assemblies) + { + foreach (var t in assembly.GetTypes()) + { + foreach (var m in t.GetMethods()) + { + var attributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true); + if (attributes.Length > 0) + { + testClasses.Add(t.FullName); + break; + } + + } + } + } + testClasses.Sort(); + return testClasses; + } + + private string GetTestsJson() + { + var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + var testsJsonFile = Path.Combine(assemblyDir, "..", "..", "..", "tests.json"); + + return File.ReadAllText(testsJsonFile); + } + + private List GetTestAssemblies() + { + var result = new List(); + var executingAssembly = Assembly.GetExecutingAssembly(); + + result.Add(executingAssembly); + + var otherAssemblies = new[] { + "Grpc.Examples.Tests", + "Grpc.HealthCheck.Tests", + "Grpc.IntegrationTesting" + }; + foreach (var assemblyName in otherAssemblies) + { + var location = executingAssembly.Location.Replace("Grpc.Core.Tests", assemblyName); + result.Add(Assembly.LoadFrom(location)); + } + return result; + } + } +} diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs index 5facb87971e..18168f99704 100644 --- a/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs @@ -140,12 +140,14 @@ namespace Grpc.IntegrationTesting } [Test] + [Ignore("TODO: see #4427")] public async Task StatusCodeAndMessage() { await InteropClient.RunStatusCodeAndMessageAsync(client); } [Test] + [Ignore("TODO: see #4427")] public void UnimplementedMethod() { InteropClient.RunUnimplementedMethod(UnimplementedService.NewClient(channel)); diff --git a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs index 2b51526c88f..3dd91b79485 100644 --- a/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs +++ b/src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs @@ -75,9 +75,10 @@ namespace Grpc.IntegrationTesting serverRunner.StopAsync().Wait(); } - // Test attribute commented out to prevent running as part of the default test suite. - //[Test] - //[Category("Performance")] + + [Test] + [Category("Performance")] + [Ignore("Prevent running on Jenkins")] public async Task ClientServerRunner() { var config = new ClientConfig diff --git a/src/csharp/tests.json b/src/csharp/tests.json new file mode 100644 index 00000000000..4aa93668ad1 --- /dev/null +++ b/src/csharp/tests.json @@ -0,0 +1,45 @@ +{ + "assemblies": [ + "Grpc.Core.Tests", + "Grpc.Examples.Tests", + "Grpc.HealthCheck.Tests", + "Grpc.IntegrationTesting" + ], + "tests": [ + "Grpc.Core.Internal.Tests.AsyncCallTest", + "Grpc.Core.Internal.Tests.ChannelArgsSafeHandleTest", + "Grpc.Core.Internal.Tests.CompletionQueueEventTest", + "Grpc.Core.Internal.Tests.CompletionQueueSafeHandleTest", + "Grpc.Core.Internal.Tests.MetadataArraySafeHandleTest", + "Grpc.Core.Internal.Tests.TimespecTest", + "Grpc.Core.Tests.CallCredentialsTest", + "Grpc.Core.Tests.CallOptionsTest", + "Grpc.Core.Tests.ChannelCredentialsTest", + "Grpc.Core.Tests.ChannelOptionsTest", + "Grpc.Core.Tests.ChannelTest", + "Grpc.Core.Tests.ClientServerTest", + "Grpc.Core.Tests.CompressionTest", + "Grpc.Core.Tests.ContextPropagationTest", + "Grpc.Core.Tests.GrpcEnvironmentTest", + "Grpc.Core.Tests.MarshallingErrorsTest", + "Grpc.Core.Tests.MetadataTest", + "Grpc.Core.Tests.NUnitVersionTest", + "Grpc.Core.Tests.PerformanceTest", + "Grpc.Core.Tests.PInvokeTest", + "Grpc.Core.Tests.ResponseHeadersTest", + "Grpc.Core.Tests.SanityTest", + "Grpc.Core.Tests.ServerTest", + "Grpc.Core.Tests.ShutdownTest", + "Grpc.Core.Tests.TimeoutsTest", + "Grpc.Core.Tests.UserAgentStringTest", + "Math.Tests.MathClientServerTest", + "Grpc.HealthCheck.Tests.HealthClientServerTest", + "Grpc.HealthCheck.Tests.HealthServiceImplTest", + "Grpc.IntegrationTesting.HeaderInterceptorTest", + "Grpc.IntegrationTesting.HistogramTest", + "Grpc.IntegrationTesting.InteropClientServerTest", + "Grpc.IntegrationTesting.MetadataCredentialsTest", + "Grpc.IntegrationTesting.RunnerClientServerTest", + "Grpc.IntegrationTesting.SslCredentialsTest" + ] +} \ No newline at end of file diff --git a/tools/run_tests/build_csharp.sh b/tools/run_tests/build_csharp.sh index 2c333517922..6737d88b273 100755 --- a/tools/run_tests/build_csharp.sh +++ b/tools/run_tests/build_csharp.sh @@ -30,7 +30,7 @@ set -ex -if [ "$CONFIG" = "dbg" ] || [ "$CONFIG" = "gcov" ] +if [ "$CONFIG" = "dbg" ] then MSBUILD_CONFIG="Debug" else diff --git a/tools/run_tests/run_csharp.bat b/tools/run_tests/run_csharp.bat index 0aa32ea5962..82eb58518ce 100644 --- a/tools/run_tests/run_csharp.bat +++ b/tools/run_tests/run_csharp.bat @@ -6,16 +6,11 @@ setlocal cd /d %~dp0\..\..\src\csharp if not "%CONFIG%" == "gcov" ( - @rem Run tests for assembly passed as 1st arg. - - @rem set UUID variable to a random GUID, we will use it to put TestResults.xml to a dedicated directory, so that parallel test runs don't collide - for /F %%i in ('powershell -Command "[guid]::NewGuid().ToString()"') do (set UUID=%%i) - - packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe /domain:None -labels "%1/bin/Debug/%1.dll" -work test-results/%UUID% || goto :error + packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe %* || goto :error ) else ( @rem Run all tests with code coverage - packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -target:"packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe" -targetdir:"." -targetargs:"/domain:None -labels Grpc.Core.Tests/bin/Debug/Grpc.Core.Tests.dll Grpc.IntegrationTesting/bin/Debug/Grpc.IntegrationTesting.dll Grpc.Examples.Tests/bin/Debug/Grpc.Examples.Tests.dll Grpc.HealthCheck.Tests/bin/Debug/Grpc.HealthCheck.Tests.dll" -filter:"+[Grpc.Core]*" -register:user -output:coverage_results.xml || goto :error + packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -target:"packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe" -targetdir:"." -targetargs:"%*" -filter:"+[Grpc.Core]*" -register:user -output:coverage_results.xml || goto :error packages\ReportGenerator.2.3.2.0\tools\ReportGenerator.exe -reports:"coverage_results.xml" -targetdir:"..\..\reports\csharp_coverage" -reporttypes:"Html;TextSummary" || goto :error diff --git a/tools/run_tests/run_csharp.sh b/tools/run_tests/run_csharp.sh index 37e86feaad3..744df07e1ca 100755 --- a/tools/run_tests/run_csharp.sh +++ b/tools/run_tests/run_csharp.sh @@ -31,38 +31,25 @@ set -ex CONFIG=${CONFIG:-opt} - NUNIT_CONSOLE="mono packages/NUnit.Runners.2.6.4/tools/nunit-console.exe" -if [ "$CONFIG" = "dbg" ] || [ "$CONFIG" = "gcov" ] -then - MSBUILD_CONFIG="Debug" -else - MSBUILD_CONFIG="Release" -fi - # change to gRPC repo root cd $(dirname $0)/../.. -root=`pwd` -export LD_LIBRARY_PATH=$root/libs/$CONFIG +# path needs to be absolute +export LD_LIBRARY_PATH=$(pwd)/libs/$CONFIG + +(cd src/csharp; $NUNIT_CONSOLE $@) if [ "$CONFIG" = "gcov" ] then - (cd src/csharp; $NUNIT_CONSOLE -labels \ - "Grpc.Core.Tests/bin/$MSBUILD_CONFIG/Grpc.Core.Tests.dll" \ - "Grpc.Examples.Tests/bin/$MSBUILD_CONFIG/Grpc.Examples.Tests.dll" \ - "Grpc.HealthCheck.Tests/bin/$MSBUILD_CONFIG/Grpc.HealthCheck.Tests.dll" \ - "Grpc.IntegrationTesting/bin/$MSBUILD_CONFIG/Grpc.IntegrationTesting.dll") - + # Generate the csharp extension coverage report gcov objs/gcov/src/csharp/ext/*.o lcov --base-directory . --directory . -c -o coverage.info lcov -e coverage.info '**/src/csharp/ext/*' -o coverage.info genhtml -o reports/csharp_ext_coverage --num-spaces 2 \ -t 'gRPC C# native extension test coverage' coverage.info \ --rc genhtml_hi_limit=95 --rc genhtml_med_limit=80 --no-prefix -else - (cd src/csharp; $NUNIT_CONSOLE -labels "$1/bin/$MSBUILD_CONFIG/$1.dll") fi diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 006f4bcdf1f..3803e8c0448 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -47,6 +47,7 @@ import tempfile import traceback import time import urllib2 +import uuid import jobset import report_utils @@ -336,26 +337,42 @@ class CSharpLanguage(object): self.platform = platform_string() def test_specs(self, config, args): - assemblies = ['Grpc.Core.Tests', - 'Grpc.Examples.Tests', - 'Grpc.HealthCheck.Tests', - 'Grpc.IntegrationTesting'] + with open('src/csharp/tests.json') as f: + tests_json = json.load(f) + assemblies = tests_json['assemblies'] + tests = tests_json['tests'] + + msbuild_config = _WINDOWS_CONFIG[config.build_config] + assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a) + for a in assemblies] + + extra_args = ['-labels'] + assembly_files + if self.platform == 'windows': - cmd = 'tools\\run_tests\\run_csharp.bat' + script_name = 'tools\\run_tests\\run_csharp.bat' + extra_args += ['-domain=None'] else: - cmd = 'tools/run_tests/run_csharp.sh' + script_name = 'tools/run_tests/run_csharp.sh' if config.build_config == 'gcov': # On Windows, we only collect C# code coverage. # On Linux, we only collect coverage for native extension. # For code coverage all tests need to run as one suite. - return [config.job_spec([cmd], None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + return [config.job_spec([script_name] + extra_args, None, + shortname='csharp.coverage', + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] else: - return [config.job_spec([cmd, assembly], - None, shortname=assembly, - environ=_FORCE_ENVIRON_FOR_WRAPPERS) - for assembly in assemblies] + specs = [] + for test in tests: + cmdline = [script_name, '-run=%s' % test] + extra_args + if self.platform == 'windows': + # use different output directory for each test to prevent + # TestResult.xml clash between parallel test runs. + cmdline += ['-work=test-result/%s' % uuid.uuid4()] + specs.append(config.job_spec(cmdline, None, + shortname='csharp.%s' % test, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)) + return specs def pre_build_steps(self): if self.platform == 'windows': @@ -509,6 +526,7 @@ _LANGUAGES = { _WINDOWS_CONFIG = { 'dbg': 'Debug', 'opt': 'Release', + 'gcov': 'Release', }