Merge pull request #4449 from jtattermusch/csharp_test_restructuring

C# test restructuring
pull/4465/head
Michael Lumish 9 years ago
commit b4d93b511a
  1. 1
      src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj
  2. 14
      src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
  3. 4
      src/csharp/Grpc.Core.Tests/PInvokeTest.cs
  4. 8
      src/csharp/Grpc.Core.Tests/PerformanceTest.cs
  5. 125
      src/csharp/Grpc.Core.Tests/SanityTest.cs
  6. 2
      src/csharp/Grpc.IntegrationTesting/InteropClientServerTest.cs
  7. 7
      src/csharp/Grpc.IntegrationTesting/RunnerClientServerTest.cs
  8. 45
      src/csharp/tests.json
  9. 2
      tools/run_tests/build_csharp.sh
  10. 9
      tools/run_tests/run_csharp.bat
  11. 23
      tools/run_tests/run_csharp.sh
  12. 42
      tools/run_tests/run_tests.py

@ -91,6 +91,7 @@
<Compile Include="ContextPropagationTest.cs" />
<Compile Include="MetadataTest.cs" />
<Compile Include="PerformanceTest.cs" />
<Compile Include="SanityTest.cs" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
<ItemGroup>

@ -167,18 +167,18 @@ namespace Grpc.Core.Internal.Tests
() => Timespec.FromDateTime(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Unspecified)));
}
// Test attribute commented out to prevent running as part of the default test suite.
// [Test]
// [Category("Performance")]
[Test]
[Category("Performance")]
[Ignore("Prevent running on Jenkins")]
public void NowBenchmark()
{
// approx Timespec.Now latency <33ns
BenchmarkUtil.RunBenchmark(10000000, 1000000000, () => { var now = Timespec.Now; });
}
// Test attribute commented out to prevent running as part of the default test suite.
// [Test]
// [Category("Performance")]
[Test]
[Category("Performance")]
[Ignore("Prevent running on Jenkins")]
public void PreciseNowBenchmark()
{
// approx Timespec.PreciseNow latency <18ns (when compiled with GRPC_TIMERS_RDTSC)

@ -59,6 +59,8 @@ namespace Grpc.Core.Tests
[Test]
public void CompletionQueueCreateDestroyBenchmark()
{
GrpcEnvironment.AddRef(); // completion queue requires gRPC environment being initialized.
BenchmarkUtil.RunBenchmark(
10, 10,
() =>
@ -66,6 +68,8 @@ namespace Grpc.Core.Tests
CompletionQueueSafeHandle cq = CompletionQueueSafeHandle.Create();
cq.Dispose();
});
GrpcEnvironment.Release();
}
/// <summary>

@ -67,10 +67,10 @@ namespace Grpc.Core.Tests
channel.ShutdownAsync().Wait();
server.ShutdownAsync().Wait();
}
// Test attribute commented out to prevent running as part of the default test suite.
//[Test]
//[Category("Performance")]
[Test]
[Category("Performance")]
[Ignore("Prevent running on Jenkins")]
public void UnaryCallPerformance()
{
var profiler = new BasicProfiler();

@ -0,0 +1,125 @@
#region Copyright notice and license
// Copyright 2015, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using Grpc.Core;
using Grpc.Core.Internal;
using Grpc.Core.Utils;
using NUnit.Framework;
namespace Grpc.Core.Tests
{
public class SanityTest
{
/// <summary>
/// Because we depend on a native library, sometimes when things go wrong, the
/// entire NUnit test process crashes. To be able to track down problems better,
/// the NUnit tests are run by run_tests.py script in a separate process per test class.
/// The list of tests to run is stored in src/csharp/tests.json.
/// This test checks that the tests.json file is up to date by discovering all the
/// existing NUnit tests in all test assemblies and comparing to contents of tests.json.
/// </summary>
[Test]
public void TestsJsonUpToDate()
{
var testClasses = DiscoverAllTestClasses();
string testsJson = GetTestsJson();
// we don't have a JSON parser at hand, but check that the test class
// name is contained in the file instead.
foreach (var className in testClasses) {
Assert.IsTrue(testsJson.Contains(className),
string.Format("Test class \"{0}\" is missing in C# tests.json file", className));
}
}
/// <summary>
/// Gets list of all test classes obtained by inspecting all the test assemblies.
/// </summary>
private List<string> DiscoverAllTestClasses()
{
var assemblies = GetTestAssemblies();
var testClasses = new List<string>();
foreach (var assembly in assemblies)
{
foreach (var t in assembly.GetTypes())
{
foreach (var m in t.GetMethods())
{
var attributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true);
if (attributes.Length > 0)
{
testClasses.Add(t.FullName);
break;
}
}
}
}
testClasses.Sort();
return testClasses;
}
private string GetTestsJson()
{
var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
var testsJsonFile = Path.Combine(assemblyDir, "..", "..", "..", "tests.json");
return File.ReadAllText(testsJsonFile);
}
private List<Assembly> GetTestAssemblies()
{
var result = new List<Assembly>();
var executingAssembly = Assembly.GetExecutingAssembly();
result.Add(executingAssembly);
var otherAssemblies = new[] {
"Grpc.Examples.Tests",
"Grpc.HealthCheck.Tests",
"Grpc.IntegrationTesting"
};
foreach (var assemblyName in otherAssemblies)
{
var location = executingAssembly.Location.Replace("Grpc.Core.Tests", assemblyName);
result.Add(Assembly.LoadFrom(location));
}
return result;
}
}
}

@ -140,12 +140,14 @@ namespace Grpc.IntegrationTesting
}
[Test]
[Ignore("TODO: see #4427")]
public async Task StatusCodeAndMessage()
{
await InteropClient.RunStatusCodeAndMessageAsync(client);
}
[Test]
[Ignore("TODO: see #4427")]
public void UnimplementedMethod()
{
InteropClient.RunUnimplementedMethod(UnimplementedService.NewClient(channel));

@ -75,9 +75,10 @@ namespace Grpc.IntegrationTesting
serverRunner.StopAsync().Wait();
}
// Test attribute commented out to prevent running as part of the default test suite.
//[Test]
//[Category("Performance")]
[Test]
[Category("Performance")]
[Ignore("Prevent running on Jenkins")]
public async Task ClientServerRunner()
{
var config = new ClientConfig

@ -0,0 +1,45 @@
{
"assemblies": [
"Grpc.Core.Tests",
"Grpc.Examples.Tests",
"Grpc.HealthCheck.Tests",
"Grpc.IntegrationTesting"
],
"tests": [
"Grpc.Core.Internal.Tests.AsyncCallTest",
"Grpc.Core.Internal.Tests.ChannelArgsSafeHandleTest",
"Grpc.Core.Internal.Tests.CompletionQueueEventTest",
"Grpc.Core.Internal.Tests.CompletionQueueSafeHandleTest",
"Grpc.Core.Internal.Tests.MetadataArraySafeHandleTest",
"Grpc.Core.Internal.Tests.TimespecTest",
"Grpc.Core.Tests.CallCredentialsTest",
"Grpc.Core.Tests.CallOptionsTest",
"Grpc.Core.Tests.ChannelCredentialsTest",
"Grpc.Core.Tests.ChannelOptionsTest",
"Grpc.Core.Tests.ChannelTest",
"Grpc.Core.Tests.ClientServerTest",
"Grpc.Core.Tests.CompressionTest",
"Grpc.Core.Tests.ContextPropagationTest",
"Grpc.Core.Tests.GrpcEnvironmentTest",
"Grpc.Core.Tests.MarshallingErrorsTest",
"Grpc.Core.Tests.MetadataTest",
"Grpc.Core.Tests.NUnitVersionTest",
"Grpc.Core.Tests.PerformanceTest",
"Grpc.Core.Tests.PInvokeTest",
"Grpc.Core.Tests.ResponseHeadersTest",
"Grpc.Core.Tests.SanityTest",
"Grpc.Core.Tests.ServerTest",
"Grpc.Core.Tests.ShutdownTest",
"Grpc.Core.Tests.TimeoutsTest",
"Grpc.Core.Tests.UserAgentStringTest",
"Math.Tests.MathClientServerTest",
"Grpc.HealthCheck.Tests.HealthClientServerTest",
"Grpc.HealthCheck.Tests.HealthServiceImplTest",
"Grpc.IntegrationTesting.HeaderInterceptorTest",
"Grpc.IntegrationTesting.HistogramTest",
"Grpc.IntegrationTesting.InteropClientServerTest",
"Grpc.IntegrationTesting.MetadataCredentialsTest",
"Grpc.IntegrationTesting.RunnerClientServerTest",
"Grpc.IntegrationTesting.SslCredentialsTest"
]
}

@ -30,7 +30,7 @@
set -ex
if [ "$CONFIG" = "dbg" ] || [ "$CONFIG" = "gcov" ]
if [ "$CONFIG" = "dbg" ]
then
MSBUILD_CONFIG="Debug"
else

@ -6,16 +6,11 @@ setlocal
cd /d %~dp0\..\..\src\csharp
if not "%CONFIG%" == "gcov" (
@rem Run tests for assembly passed as 1st arg.
@rem set UUID variable to a random GUID, we will use it to put TestResults.xml to a dedicated directory, so that parallel test runs don't collide
for /F %%i in ('powershell -Command "[guid]::NewGuid().ToString()"') do (set UUID=%%i)
packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe /domain:None -labels "%1/bin/Debug/%1.dll" -work test-results/%UUID% || goto :error
packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe %* || goto :error
) else (
@rem Run all tests with code coverage
packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -target:"packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe" -targetdir:"." -targetargs:"/domain:None -labels Grpc.Core.Tests/bin/Debug/Grpc.Core.Tests.dll Grpc.IntegrationTesting/bin/Debug/Grpc.IntegrationTesting.dll Grpc.Examples.Tests/bin/Debug/Grpc.Examples.Tests.dll Grpc.HealthCheck.Tests/bin/Debug/Grpc.HealthCheck.Tests.dll" -filter:"+[Grpc.Core]*" -register:user -output:coverage_results.xml || goto :error
packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -target:"packages\NUnit.Runners.2.6.4\tools\nunit-console-x86.exe" -targetdir:"." -targetargs:"%*" -filter:"+[Grpc.Core]*" -register:user -output:coverage_results.xml || goto :error
packages\ReportGenerator.2.3.2.0\tools\ReportGenerator.exe -reports:"coverage_results.xml" -targetdir:"..\..\reports\csharp_coverage" -reporttypes:"Html;TextSummary" || goto :error

@ -31,38 +31,25 @@
set -ex
CONFIG=${CONFIG:-opt}
NUNIT_CONSOLE="mono packages/NUnit.Runners.2.6.4/tools/nunit-console.exe"
if [ "$CONFIG" = "dbg" ] || [ "$CONFIG" = "gcov" ]
then
MSBUILD_CONFIG="Debug"
else
MSBUILD_CONFIG="Release"
fi
# change to gRPC repo root
cd $(dirname $0)/../..
root=`pwd`
export LD_LIBRARY_PATH=$root/libs/$CONFIG
# path needs to be absolute
export LD_LIBRARY_PATH=$(pwd)/libs/$CONFIG
(cd src/csharp; $NUNIT_CONSOLE $@)
if [ "$CONFIG" = "gcov" ]
then
(cd src/csharp; $NUNIT_CONSOLE -labels \
"Grpc.Core.Tests/bin/$MSBUILD_CONFIG/Grpc.Core.Tests.dll" \
"Grpc.Examples.Tests/bin/$MSBUILD_CONFIG/Grpc.Examples.Tests.dll" \
"Grpc.HealthCheck.Tests/bin/$MSBUILD_CONFIG/Grpc.HealthCheck.Tests.dll" \
"Grpc.IntegrationTesting/bin/$MSBUILD_CONFIG/Grpc.IntegrationTesting.dll")
# Generate the csharp extension coverage report
gcov objs/gcov/src/csharp/ext/*.o
lcov --base-directory . --directory . -c -o coverage.info
lcov -e coverage.info '**/src/csharp/ext/*' -o coverage.info
genhtml -o reports/csharp_ext_coverage --num-spaces 2 \
-t 'gRPC C# native extension test coverage' coverage.info \
--rc genhtml_hi_limit=95 --rc genhtml_med_limit=80 --no-prefix
else
(cd src/csharp; $NUNIT_CONSOLE -labels "$1/bin/$MSBUILD_CONFIG/$1.dll")
fi

@ -47,6 +47,7 @@ import tempfile
import traceback
import time
import urllib2
import uuid
import jobset
import report_utils
@ -336,26 +337,42 @@ class CSharpLanguage(object):
self.platform = platform_string()
def test_specs(self, config, args):
assemblies = ['Grpc.Core.Tests',
'Grpc.Examples.Tests',
'Grpc.HealthCheck.Tests',
'Grpc.IntegrationTesting']
with open('src/csharp/tests.json') as f:
tests_json = json.load(f)
assemblies = tests_json['assemblies']
tests = tests_json['tests']
msbuild_config = _WINDOWS_CONFIG[config.build_config]
assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a)
for a in assemblies]
extra_args = ['-labels'] + assembly_files
if self.platform == 'windows':
cmd = 'tools\\run_tests\\run_csharp.bat'
script_name = 'tools\\run_tests\\run_csharp.bat'
extra_args += ['-domain=None']
else:
cmd = 'tools/run_tests/run_csharp.sh'
script_name = 'tools/run_tests/run_csharp.sh'
if config.build_config == 'gcov':
# On Windows, we only collect C# code coverage.
# On Linux, we only collect coverage for native extension.
# For code coverage all tests need to run as one suite.
return [config.job_spec([cmd], None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
return [config.job_spec([script_name] + extra_args, None,
shortname='csharp.coverage',
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
else:
return [config.job_spec([cmd, assembly],
None, shortname=assembly,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)
for assembly in assemblies]
specs = []
for test in tests:
cmdline = [script_name, '-run=%s' % test] + extra_args
if self.platform == 'windows':
# use different output directory for each test to prevent
# TestResult.xml clash between parallel test runs.
cmdline += ['-work=test-result/%s' % uuid.uuid4()]
specs.append(config.job_spec(cmdline, None,
shortname='csharp.%s' % test,
environ=_FORCE_ENVIRON_FOR_WRAPPERS))
return specs
def pre_build_steps(self):
if self.platform == 'windows':
@ -509,6 +526,7 @@ _LANGUAGES = {
_WINDOWS_CONFIG = {
'dbg': 'Debug',
'opt': 'Release',
'gcov': 'Release',
}

Loading…
Cancel
Save