From 03c010611bbef48461e4994c509f6887a7b63d00 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 11 Dec 2015 14:28:56 -0800 Subject: [PATCH] split up C# tests into separate processes --- src/csharp/tests.json | 45 +++++++++++++++++++++++++++++++++ tools/run_tests/build_csharp.sh | 2 +- tools/run_tests/run_csharp.bat | 9 ++----- tools/run_tests/run_csharp.sh | 23 ++++------------- tools/run_tests/run_tests.py | 42 +++++++++++++++++++++--------- 5 files changed, 83 insertions(+), 38 deletions(-) create mode 100644 src/csharp/tests.json diff --git a/src/csharp/tests.json b/src/csharp/tests.json new file mode 100644 index 00000000000..4aa93668ad1 --- /dev/null +++ b/src/csharp/tests.json @@ -0,0 +1,45 @@ +{ + "assemblies": [ + "Grpc.Core.Tests", + "Grpc.Examples.Tests", + "Grpc.HealthCheck.Tests", + "Grpc.IntegrationTesting" + ], + "tests": [ + "Grpc.Core.Internal.Tests.AsyncCallTest", + "Grpc.Core.Internal.Tests.ChannelArgsSafeHandleTest", + "Grpc.Core.Internal.Tests.CompletionQueueEventTest", + "Grpc.Core.Internal.Tests.CompletionQueueSafeHandleTest", + "Grpc.Core.Internal.Tests.MetadataArraySafeHandleTest", + "Grpc.Core.Internal.Tests.TimespecTest", + "Grpc.Core.Tests.CallCredentialsTest", + "Grpc.Core.Tests.CallOptionsTest", + "Grpc.Core.Tests.ChannelCredentialsTest", + "Grpc.Core.Tests.ChannelOptionsTest", + "Grpc.Core.Tests.ChannelTest", + "Grpc.Core.Tests.ClientServerTest", + "Grpc.Core.Tests.CompressionTest", + "Grpc.Core.Tests.ContextPropagationTest", + "Grpc.Core.Tests.GrpcEnvironmentTest", + "Grpc.Core.Tests.MarshallingErrorsTest", + "Grpc.Core.Tests.MetadataTest", + "Grpc.Core.Tests.NUnitVersionTest", + "Grpc.Core.Tests.PerformanceTest", + "Grpc.Core.Tests.PInvokeTest", + "Grpc.Core.Tests.ResponseHeadersTest", + "Grpc.Core.Tests.SanityTest", + "Grpc.Core.Tests.ServerTest", + "Grpc.Core.Tests.ShutdownTest", + "Grpc.Core.Tests.TimeoutsTest", + "Grpc.Core.Tests.UserAgentStringTest", + "Math.Tests.MathClientServerTest", + "Grpc.HealthCheck.Tests.HealthClientServerTest", + "Grpc.HealthCheck.Tests.HealthServiceImplTest", + "Grpc.IntegrationTesting.HeaderInterceptorTest", + "Grpc.IntegrationTesting.HistogramTest", + "Grpc.IntegrationTesting.InteropClientServerTest", + "Grpc.IntegrationTesting.MetadataCredentialsTest", + "Grpc.IntegrationTesting.RunnerClientServerTest", + "Grpc.IntegrationTesting.SslCredentialsTest" + ] +} \ No newline at end of file diff --git a/tools/run_tests/build_csharp.sh b/tools/run_tests/build_csharp.sh index 2c333517922..6737d88b273 100755 --- a/tools/run_tests/build_csharp.sh +++ b/tools/run_tests/build_csharp.sh @@ -30,7 +30,7 @@ set -ex -if [ "$CONFIG" = "dbg" ] || [ "$CONFIG" = "gcov" ] +if [ "$CONFIG" = "dbg" ] then MSBUILD_CONFIG="Debug" else diff --git a/tools/run_tests/run_csharp.bat b/tools/run_tests/run_csharp.bat index 0aa32ea5962..82eb58518ce 100644 --- a/tools/run_tests/run_csharp.bat +++ b/tools/run_tests/run_csharp.bat @@ -6,16 +6,11 @@ setlocal cd /d %~dp0\..\..\src\csharp if not "%CONFIG%" == "gcov" ( - @rem Run tests for assembly passed as 1st arg. - - @rem set UUID variable to a random GUID, we will use it to put TestResults.xml to a dedicated directory, so that parallel test runs don't collide - for /F %%i in ('powershell -Command "[guid]::NewGuid().ToString()"') do (set UUID=%%i) - - packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe /domain:None -labels "%1/bin/Debug/%1.dll" -work test-results/%UUID% || goto :error + packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe %* || goto :error ) else ( @rem Run all tests with code coverage - packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -target:"packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe" -targetdir:"." -targetargs:"/domain:None -labels Grpc.Core.Tests/bin/Debug/Grpc.Core.Tests.dll Grpc.IntegrationTesting/bin/Debug/Grpc.IntegrationTesting.dll Grpc.Examples.Tests/bin/Debug/Grpc.Examples.Tests.dll Grpc.HealthCheck.Tests/bin/Debug/Grpc.HealthCheck.Tests.dll" -filter:"+[Grpc.Core]*" -register:user -output:coverage_results.xml || goto :error + packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -target:"packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe" -targetdir:"." -targetargs:"%*" -filter:"+[Grpc.Core]*" -register:user -output:coverage_results.xml || goto :error packages\ReportGenerator.2.3.2.0\tools\ReportGenerator.exe -reports:"coverage_results.xml" -targetdir:"..\..\reports\csharp_coverage" -reporttypes:"Html;TextSummary" || goto :error diff --git a/tools/run_tests/run_csharp.sh b/tools/run_tests/run_csharp.sh index 37e86feaad3..744df07e1ca 100755 --- a/tools/run_tests/run_csharp.sh +++ b/tools/run_tests/run_csharp.sh @@ -31,38 +31,25 @@ set -ex CONFIG=${CONFIG:-opt} - NUNIT_CONSOLE="mono packages/NUnit.Runners.2.6.4/tools/nunit-console.exe" -if [ "$CONFIG" = "dbg" ] || [ "$CONFIG" = "gcov" ] -then - MSBUILD_CONFIG="Debug" -else - MSBUILD_CONFIG="Release" -fi - # change to gRPC repo root cd $(dirname $0)/../.. -root=`pwd` -export LD_LIBRARY_PATH=$root/libs/$CONFIG +# path needs to be absolute +export LD_LIBRARY_PATH=$(pwd)/libs/$CONFIG + +(cd src/csharp; $NUNIT_CONSOLE $@) if [ "$CONFIG" = "gcov" ] then - (cd src/csharp; $NUNIT_CONSOLE -labels \ - "Grpc.Core.Tests/bin/$MSBUILD_CONFIG/Grpc.Core.Tests.dll" \ - "Grpc.Examples.Tests/bin/$MSBUILD_CONFIG/Grpc.Examples.Tests.dll" \ - "Grpc.HealthCheck.Tests/bin/$MSBUILD_CONFIG/Grpc.HealthCheck.Tests.dll" \ - "Grpc.IntegrationTesting/bin/$MSBUILD_CONFIG/Grpc.IntegrationTesting.dll") - + # Generate the csharp extension coverage report gcov objs/gcov/src/csharp/ext/*.o lcov --base-directory . --directory . -c -o coverage.info lcov -e coverage.info '**/src/csharp/ext/*' -o coverage.info genhtml -o reports/csharp_ext_coverage --num-spaces 2 \ -t 'gRPC C# native extension test coverage' coverage.info \ --rc genhtml_hi_limit=95 --rc genhtml_med_limit=80 --no-prefix -else - (cd src/csharp; $NUNIT_CONSOLE -labels "$1/bin/$MSBUILD_CONFIG/$1.dll") fi diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 006f4bcdf1f..3803e8c0448 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -47,6 +47,7 @@ import tempfile import traceback import time import urllib2 +import uuid import jobset import report_utils @@ -336,26 +337,42 @@ class CSharpLanguage(object): self.platform = platform_string() def test_specs(self, config, args): - assemblies = ['Grpc.Core.Tests', - 'Grpc.Examples.Tests', - 'Grpc.HealthCheck.Tests', - 'Grpc.IntegrationTesting'] + with open('src/csharp/tests.json') as f: + tests_json = json.load(f) + assemblies = tests_json['assemblies'] + tests = tests_json['tests'] + + msbuild_config = _WINDOWS_CONFIG[config.build_config] + assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a) + for a in assemblies] + + extra_args = ['-labels'] + assembly_files + if self.platform == 'windows': - cmd = 'tools\\run_tests\\run_csharp.bat' + script_name = 'tools\\run_tests\\run_csharp.bat' + extra_args += ['-domain=None'] else: - cmd = 'tools/run_tests/run_csharp.sh' + script_name = 'tools/run_tests/run_csharp.sh' if config.build_config == 'gcov': # On Windows, we only collect C# code coverage. # On Linux, we only collect coverage for native extension. # For code coverage all tests need to run as one suite. - return [config.job_spec([cmd], None, - environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + return [config.job_spec([script_name] + extra_args, None, + shortname='csharp.coverage', + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] else: - return [config.job_spec([cmd, assembly], - None, shortname=assembly, - environ=_FORCE_ENVIRON_FOR_WRAPPERS) - for assembly in assemblies] + specs = [] + for test in tests: + cmdline = [script_name, '-run=%s' % test] + extra_args + if self.platform == 'windows': + # use different output directory for each test to prevent + # TestResult.xml clash between parallel test runs. + cmdline += ['-work=test-result/%s' % uuid.uuid4()] + specs.append(config.job_spec(cmdline, None, + shortname='csharp.%s' % test, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)) + return specs def pre_build_steps(self): if self.platform == 'windows': @@ -509,6 +526,7 @@ _LANGUAGES = { _WINDOWS_CONFIG = { 'dbg': 'Debug', 'opt': 'Release', + 'gcov': 'Release', }