From 6bf1de9ab2b324156d766a2f2da14a5d75eae379 Mon Sep 17 00:00:00 2001 From: vjpai Date: Mon, 2 Nov 2015 14:48:57 -0800 Subject: [PATCH 1/6] Mark a method with GRPC_OVERRIDE to avoid compiler warning --- test/cpp/end2end/mock_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cpp/end2end/mock_test.cc b/test/cpp/end2end/mock_test.cc index 9c35fede8f6..80057d893e1 100644 --- a/test/cpp/end2end/mock_test.cc +++ b/test/cpp/end2end/mock_test.cc @@ -62,7 +62,7 @@ template class MockClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface { public: - void WaitForInitialMetadata() {} + void WaitForInitialMetadata() GRPC_OVERRIDE {} bool Read(R* msg) GRPC_OVERRIDE { return true; } bool Write(const W& msg) GRPC_OVERRIDE { return true; } bool WritesDone() GRPC_OVERRIDE { return true; } @@ -73,7 +73,7 @@ class MockClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface { public: MockClientReaderWriter() : writes_done_(false) {} - void WaitForInitialMetadata() {} + void WaitForInitialMetadata() GRPC_OVERRIDE {} bool Read(EchoResponse* msg) GRPC_OVERRIDE { if (writes_done_) return false; msg->set_message(last_message_); From 452ca9b912cf1173d901dc7ef0fcc4098d0ea551 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 29 Oct 2015 10:38:03 -0700 Subject: [PATCH 2/6] add profiling support --- .../Grpc.Core.Tests/ClientServerTest.cs | 14 +- .../Grpc.Core.Tests/Grpc.Core.Tests.csproj | 1 + .../Grpc.Core.Tests/Internal/TimespecTest.cs | 19 +++ src/csharp/Grpc.Core.Tests/PerformanceTest.cs | 99 +++++++++++++ src/csharp/Grpc.Core/Grpc.Core.csproj | 7 + src/csharp/Grpc.Core/Internal/AsyncCall.cs | 106 ++++++++------ .../Grpc.Core/Internal/AsyncCallBase.cs | 42 ++++-- .../Grpc.Core/Internal/CallSafeHandle.cs | 8 +- .../Grpc.Core/Internal/ChannelSafeHandle.cs | 14 +- .../Internal/CompletionQueueSafeHandle.cs | 6 +- src/csharp/Grpc.Core/Internal/Enums.cs | 3 + .../Internal/MetadataArraySafeHandle.cs | 16 ++- src/csharp/Grpc.Core/Internal/Timespec.cs | 13 ++ src/csharp/Grpc.Core/Profiling/IProfiler.cs | 47 +++++++ .../Grpc.Core/Profiling/ProfilerEntry.cs | 87 ++++++++++++ .../Grpc.Core/Profiling/ProfilerScope.cs | 60 ++++++++ src/csharp/Grpc.Core/Profiling/Profilers.cs | 131 ++++++++++++++++++ 17 files changed, 587 insertions(+), 86 deletions(-) create mode 100644 src/csharp/Grpc.Core.Tests/PerformanceTest.cs create mode 100644 src/csharp/Grpc.Core/Profiling/IProfiler.cs create mode 100644 src/csharp/Grpc.Core/Profiling/ProfilerEntry.cs create mode 100644 src/csharp/Grpc.Core/Profiling/ProfilerScope.cs create mode 100644 src/csharp/Grpc.Core/Profiling/Profilers.cs diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs index e58528ff50e..25a5a27c8e3 100644 --- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs @@ -38,6 +38,7 @@ using System.Threading; using System.Threading.Tasks; using Grpc.Core; using Grpc.Core.Internal; +using Grpc.Core.Profiling; using Grpc.Core.Utils; using NUnit.Framework; @@ -200,19 +201,6 @@ namespace Grpc.Core.Tests Assert.AreEqual(headers[1].Key, trailers[1].Key); CollectionAssert.AreEqual(headers[1].ValueBytes, trailers[1].ValueBytes); } - - [Test] - public void UnaryCallPerformance() - { - helper.UnaryHandler = new UnaryServerMethod(async (request, context) => - { - return request; - }); - - var callDetails = helper.CreateUnaryCall(); - BenchmarkUtil.RunBenchmark(1, 10, - () => { Calls.BlockingUnaryCall(callDetails, "ABC"); }); - } [Test] public void UnknownMethodHandler() diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 91d072ababe..e5ffa319895 100644 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -88,6 +88,7 @@ + diff --git a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs index 874df02baa0..9be5450d810 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs @@ -34,6 +34,7 @@ using System; using System.Runtime.InteropServices; using Grpc.Core.Internal; +using Grpc.Core.Utils; using NUnit.Framework; namespace Grpc.Core.Internal.Tests @@ -198,5 +199,23 @@ namespace Grpc.Core.Internal.Tests Console.WriteLine("Test cannot be run on this platform, skipping the test."); } } + + // Test attribute commented out to prevent running as part of the default test suite. + // [Test] + // [Category("Performance")] + public void NowBenchmark() + { + // approx Timespec.Now latency <33ns + BenchmarkUtil.RunBenchmark(10000000, 1000000000, () => { var now = Timespec.Now; }); + } + + // Test attribute commented out to prevent running as part of the default test suite. + // [Test] + // [Category("Performance")] + public void PreciseNowBenchmark() + { + // approx Timespec.PreciseNow latency <18ns (when compiled with GRPC_TIMERS_RDTSC) + BenchmarkUtil.RunBenchmark(10000000, 1000000000, () => { var now = Timespec.PreciseNow; }); + } } } diff --git a/src/csharp/Grpc.Core.Tests/PerformanceTest.cs b/src/csharp/Grpc.Core.Tests/PerformanceTest.cs new file mode 100644 index 00000000000..5516cd33774 --- /dev/null +++ b/src/csharp/Grpc.Core.Tests/PerformanceTest.cs @@ -0,0 +1,99 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Grpc.Core; +using Grpc.Core.Internal; +using Grpc.Core.Profiling; +using Grpc.Core.Utils; +using NUnit.Framework; + +namespace Grpc.Core.Tests +{ + public class PerformanceTest + { + const string Host = "127.0.0.1"; + + MockServiceHelper helper; + Server server; + Channel channel; + + [SetUp] + public void Init() + { + helper = new MockServiceHelper(Host); + server = helper.GetServer(); + server.Start(); + channel = helper.GetChannel(); + } + + [TearDown] + public void Cleanup() + { + channel.ShutdownAsync().Wait(); + server.ShutdownAsync().Wait(); + } + + // Test attribute commented out to prevent running as part of the default test suite. + //[Test] + //[Category("Performance")] + public void UnaryCallPerformance() + { + var profiler = new BasicProfiler(); + Profilers.SetForCurrentThread(profiler); + + helper.UnaryHandler = new UnaryServerMethod(async (request, context) => + { + return request; + }); + + var callDetails = helper.CreateUnaryCall(); + for(int i = 0; i < 3000; i++) + { + Calls.BlockingUnaryCall(callDetails, "ABC"); + } + + profiler.Reset(); + + for(int i = 0; i < 3000; i++) + { + Calls.BlockingUnaryCall(callDetails, "ABC"); + } + profiler.Dump("latency_trace_csharp.txt"); + } + } +} diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 92d4e19eac2..0aab7bdd8ad 100644 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -119,6 +119,10 @@ + + + + @@ -150,4 +154,7 @@ + + + \ No newline at end of file diff --git a/src/csharp/Grpc.Core/Internal/AsyncCall.cs b/src/csharp/Grpc.Core/Internal/AsyncCall.cs index 800462c8540..e3ecc472826 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCall.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCall.cs @@ -39,6 +39,7 @@ using System.Threading; using System.Threading.Tasks; using Grpc.Core.Internal; using Grpc.Core.Logging; +using Grpc.Core.Profiling; using Grpc.Core.Utils; namespace Grpc.Core.Internal @@ -87,6 +88,9 @@ namespace Grpc.Core.Internal /// public TResponse UnaryCall(TRequest msg) { + var profiler = Profilers.ForCurrentThread(); + + using (profiler.NewScope("AsyncCall.UnaryCall")) using (CompletionQueueSafeHandle cq = CompletionQueueSafeHandle.Create()) { byte[] payload = UnsafeSerialize(msg); @@ -104,24 +108,26 @@ namespace Grpc.Core.Internal } using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers)) + using (var ctx = BatchContextSafeHandle.Create()) { - using (var ctx = BatchContextSafeHandle.Create()) - { - call.StartUnary(ctx, payload, metadataArray, GetWriteFlagsForCall()); - var ev = cq.Pluck(ctx.Handle); + call.StartUnary(ctx, payload, metadataArray, GetWriteFlagsForCall()); + + var ev = cq.Pluck(ctx.Handle); - bool success = (ev.success != 0); - try + bool success = (ev.success != 0); + try + { + using (profiler.NewScope("AsyncCall.UnaryCall.HandleBatch")) { HandleUnaryResponse(success, ctx.GetReceivedStatusOnClient(), ctx.GetReceivedMessage(), ctx.GetReceivedInitialMetadata()); } - catch (Exception e) - { - Logger.Error(e, "Exception occured while invoking completion delegate."); - } + } + catch (Exception e) + { + Logger.Error(e, "Exception occured while invoking completion delegate."); } } - + // Once the blocking call returns, the result should be available synchronously. // Note that GetAwaiter().GetResult() doesn't wrap exceptions in AggregateException. return unaryResponseTcs.Task.GetAwaiter().GetResult(); @@ -329,27 +335,35 @@ namespace Grpc.Core.Internal private void Initialize(CompletionQueueSafeHandle cq) { - var call = CreateNativeCall(cq); - details.Channel.AddCallReference(this); - InitializeInternal(call); - RegisterCancellationCallback(); + using (Profilers.ForCurrentThread().NewScope("AsyncCall.Initialize")) + { + var call = CreateNativeCall(cq); + + details.Channel.AddCallReference(this); + InitializeInternal(call); + RegisterCancellationCallback(); + } } private INativeCall CreateNativeCall(CompletionQueueSafeHandle cq) { - if (injectedNativeCall != null) - { - return injectedNativeCall; // allows injecting a mock INativeCall in tests. - } + using (Profilers.ForCurrentThread().NewScope("AsyncCall.CreateNativeCall")) + { + if (injectedNativeCall != null) + { + return injectedNativeCall; // allows injecting a mock INativeCall in tests. + } - var parentCall = details.Options.PropagationToken != null ? details.Options.PropagationToken.ParentCall : CallSafeHandle.NullInstance; + var parentCall = details.Options.PropagationToken != null ? details.Options.PropagationToken.ParentCall : CallSafeHandle.NullInstance; - var credentials = details.Options.Credentials; - using (var nativeCredentials = credentials != null ? credentials.ToNativeCredentials() : null) - { - return details.Channel.Handle.CreateCall(environment.CompletionRegistry, - parentCall, ContextPropagationToken.DefaultMask, cq, - details.Method, details.Host, Timespec.FromDateTime(details.Options.Deadline.Value), nativeCredentials); + var credentials = details.Options.Credentials; + using (var nativeCredentials = credentials != null ? credentials.ToNativeCredentials() : null) + { + var result = details.Channel.Handle.CreateCall(environment.CompletionRegistry, + parentCall, ContextPropagationToken.DefaultMask, cq, + details.Method, details.Host, Timespec.FromDateTime(details.Options.Deadline.Value), nativeCredentials); + return result; + } } } @@ -385,33 +399,37 @@ namespace Grpc.Core.Internal /// private void HandleUnaryResponse(bool success, ClientSideStatus receivedStatus, byte[] receivedMessage, Metadata responseHeaders) { - TResponse msg = default(TResponse); - var deserializeException = success ? TryDeserialize(receivedMessage, out msg) : null; - - lock (myLock) + using (Profilers.ForCurrentThread().NewScope("AsyncCall.HandleUnaryResponse")) { - finished = true; + TResponse msg = default(TResponse); + var deserializeException = success ? TryDeserialize(receivedMessage, out msg) : null; - if (deserializeException != null && receivedStatus.Status.StatusCode == StatusCode.OK) + lock (myLock) { - receivedStatus = new ClientSideStatus(DeserializeResponseFailureStatus, receivedStatus.Trailers); + finished = true; + + if (deserializeException != null && receivedStatus.Status.StatusCode == StatusCode.OK) + { + receivedStatus = new ClientSideStatus(DeserializeResponseFailureStatus, receivedStatus.Trailers); + } + finishedStatus = receivedStatus; + + ReleaseResourcesIfPossible(); + } - finishedStatus = receivedStatus; - ReleaseResourcesIfPossible(); - } + responseHeadersTcs.SetResult(responseHeaders); - responseHeadersTcs.SetResult(responseHeaders); + var status = receivedStatus.Status; - var status = receivedStatus.Status; + if (!success || status.StatusCode != StatusCode.OK) + { + unaryResponseTcs.SetException(new RpcException(status)); + return; + } - if (!success || status.StatusCode != StatusCode.OK) - { - unaryResponseTcs.SetException(new RpcException(status)); - return; + unaryResponseTcs.SetResult(msg); } - - unaryResponseTcs.SetResult(msg); } /// diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs index 3e2c57c9b5b..953f61aa1ea 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs @@ -41,6 +41,7 @@ using System.Threading.Tasks; using Grpc.Core.Internal; using Grpc.Core.Logging; +using Grpc.Core.Profiling; using Grpc.Core.Utils; namespace Grpc.Core.Internal @@ -167,16 +168,19 @@ namespace Grpc.Core.Internal /// protected bool ReleaseResourcesIfPossible() { - if (!disposed && call != null) + using (Profilers.ForCurrentThread().NewScope("AsyncCallBase.ReleaseResourcesIfPossible")) { - bool noMoreSendCompletions = sendCompletionDelegate == null && (halfcloseRequested || cancelRequested || finished); - if (noMoreSendCompletions && readingDone && finished) + if (!disposed && call != null) { - ReleaseResources(); - return true; + bool noMoreSendCompletions = sendCompletionDelegate == null && (halfcloseRequested || cancelRequested || finished); + if (noMoreSendCompletions && readingDone && finished) + { + ReleaseResources(); + return true; + } } + return false; } - return false; } protected abstract bool IsClient @@ -228,7 +232,10 @@ namespace Grpc.Core.Internal protected byte[] UnsafeSerialize(TWrite msg) { - return serializer(msg); + using (Profilers.ForCurrentThread().NewScope("AsyncCallBase.UnsafeSerialize")) + { + return serializer(msg); + } } protected Exception TrySerialize(TWrite msg, out byte[] payload) @@ -247,15 +254,20 @@ namespace Grpc.Core.Internal protected Exception TryDeserialize(byte[] payload, out TRead msg) { - try - { - msg = deserializer(payload); - return null; - } - catch (Exception e) + using (Profilers.ForCurrentThread().NewScope("AsyncCallBase.TryDeserialize")) { - msg = default(TRead); - return e; + try + { + + msg = deserializer(payload); + return null; + + } + catch (Exception e) + { + msg = default(TRead); + return e; + } } } diff --git a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs index 0be7a4dd3a1..ddeedebd117 100644 --- a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs +++ b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs @@ -34,6 +34,7 @@ using System.Diagnostics; using System.Runtime.InteropServices; using Grpc.Core; using Grpc.Core.Utils; +using Grpc.Core.Profiling; namespace Grpc.Core.Internal { @@ -131,8 +132,11 @@ namespace Grpc.Core.Internal public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags) { - grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags) - .CheckOk(); + using (Profilers.ForCurrentThread().NewScope("CallSafeHandle.StartUnary")) + { + grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags) + .CheckOk(); + } } public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray) diff --git a/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs b/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs index d270d77526f..5f9169bcb2f 100644 --- a/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs +++ b/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs @@ -32,6 +32,7 @@ using System; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; +using Grpc.Core.Profiling; namespace Grpc.Core.Internal { @@ -84,13 +85,16 @@ namespace Grpc.Core.Internal public CallSafeHandle CreateCall(CompletionRegistry registry, CallSafeHandle parentCall, ContextPropagationFlags propagationMask, CompletionQueueSafeHandle cq, string method, string host, Timespec deadline, CredentialsSafeHandle credentials) { - var result = grpcsharp_channel_create_call(this, parentCall, propagationMask, cq, method, host, deadline); - if (credentials != null) + using (Profilers.ForCurrentThread().NewScope("ChannelSafeHandle.CreateCall")) { - result.SetCredentials(credentials); + var result = grpcsharp_channel_create_call(this, parentCall, propagationMask, cq, method, host, deadline); + if (credentials != null) + { + result.SetCredentials(credentials); + } + result.SetCompletionRegistry(registry); + return result; } - result.SetCompletionRegistry(registry); - return result; } public ChannelState CheckConnectivityState(bool tryToConnect) diff --git a/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs b/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs index f7a3471bb4b..9de2bc7950b 100644 --- a/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs +++ b/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs @@ -31,6 +31,7 @@ using System; using System.Runtime.InteropServices; using System.Threading.Tasks; +using Grpc.Core.Profiling; namespace Grpc.Core.Internal { @@ -70,7 +71,10 @@ namespace Grpc.Core.Internal public CompletionQueueEvent Pluck(IntPtr tag) { - return grpcsharp_completion_queue_pluck(this, tag); + using (Profilers.ForCurrentThread().NewScope("CompletionQueueSafeHandle.Pluck")) + { + return grpcsharp_completion_queue_pluck(this, tag); + } } public void Shutdown() diff --git a/src/csharp/Grpc.Core/Internal/Enums.cs b/src/csharp/Grpc.Core/Internal/Enums.cs index 185098160b6..b0eab2001bc 100644 --- a/src/csharp/Grpc.Core/Internal/Enums.cs +++ b/src/csharp/Grpc.Core/Internal/Enums.cs @@ -102,6 +102,9 @@ namespace Grpc.Core.Internal /* Realtime clock */ Realtime, + /* Precise clock good for performance profiling. */ + Precise, + /* Timespan - the distance between two time points */ Timespan } diff --git a/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs b/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs index 31b834c979a..ed1bd244980 100644 --- a/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs +++ b/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs @@ -31,6 +31,7 @@ using System; using System.Runtime.InteropServices; using System.Threading.Tasks; +using Grpc.Core.Profiling; namespace Grpc.Core.Internal { @@ -66,14 +67,17 @@ namespace Grpc.Core.Internal public static MetadataArraySafeHandle Create(Metadata metadata) { - // TODO(jtattermusch): we might wanna check that the metadata is readonly - var metadataArray = grpcsharp_metadata_array_create(new UIntPtr((ulong)metadata.Count)); - for (int i = 0; i < metadata.Count; i++) + using (Profilers.ForCurrentThread().NewScope("MetadataArraySafeHandle.Create")) { - var valueBytes = metadata[i].GetSerializedValueUnsafe(); - grpcsharp_metadata_array_add(metadataArray, metadata[i].Key, valueBytes, new UIntPtr((ulong)valueBytes.Length)); + // TODO(jtattermusch): we might wanna check that the metadata is readonly + var metadataArray = grpcsharp_metadata_array_create(new UIntPtr((ulong)metadata.Count)); + for (int i = 0; i < metadata.Count; i++) + { + var valueBytes = metadata[i].GetSerializedValueUnsafe(); + grpcsharp_metadata_array_add(metadataArray, metadata[i].Key, valueBytes, new UIntPtr((ulong)valueBytes.Length)); + } + return metadataArray; } - return metadataArray; } /// diff --git a/src/csharp/Grpc.Core/Internal/Timespec.cs b/src/csharp/Grpc.Core/Internal/Timespec.cs index daf85d5f61d..38fc067d9f4 100644 --- a/src/csharp/Grpc.Core/Internal/Timespec.cs +++ b/src/csharp/Grpc.Core/Internal/Timespec.cs @@ -239,6 +239,19 @@ namespace Grpc.Core.Internal } } + /// + /// Gets current timestamp using GPRClockType.Precise. + /// Only available internally because core needs to be compiled with + /// GRPC_TIMERS_RDTSC support for this to use RDTSC. + /// + internal static Timespec PreciseNow + { + get + { + return gprsharp_now(GPRClockType.Precise); + } + } + internal static int NativeSize { get diff --git a/src/csharp/Grpc.Core/Profiling/IProfiler.cs b/src/csharp/Grpc.Core/Profiling/IProfiler.cs new file mode 100644 index 00000000000..c426c365d2d --- /dev/null +++ b/src/csharp/Grpc.Core/Profiling/IProfiler.cs @@ -0,0 +1,47 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; +using System.IO; +using System.Threading; +using Grpc.Core.Internal; + +namespace Grpc.Core.Profiling +{ + internal interface IProfiler + { + void Begin(string tag); + void End(string tag); + void Mark(string tag); + } +} diff --git a/src/csharp/Grpc.Core/Profiling/ProfilerEntry.cs b/src/csharp/Grpc.Core/Profiling/ProfilerEntry.cs new file mode 100644 index 00000000000..5cc4c3c0542 --- /dev/null +++ b/src/csharp/Grpc.Core/Profiling/ProfilerEntry.cs @@ -0,0 +1,87 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; +using System.IO; +using System.Threading; +using Grpc.Core.Internal; + +namespace Grpc.Core.Profiling +{ + internal struct ProfilerEntry + { + public enum Type { + BEGIN, + END, + MARK + } + + public ProfilerEntry(Timespec timespec, Type type, string tag) + { + this.timespec = timespec; + this.type = type; + this.tag = tag; + } + + public Timespec timespec; + public Type type; + public string tag; + + public override string ToString() + { + // mimic the output format used by C core. + return string.Format( + "{{\"t\": {0}.{1}, \"thd\":\"unknown\", \"type\": \"{2}\", \"tag\": \"{3}\", " + + "\"file\": \"unknown\", \"line\": 0, \"imp\": 0}}", + timespec.TimevalSeconds, timespec.TimevalNanos.ToString("D9"), + GetTypeAbbreviation(type), tag); + } + + internal static string GetTypeAbbreviation(Type type) + { + switch (type) + { + case Type.BEGIN: + return "{"; + + case Type.END: + return "}"; + + case Type.MARK: + return "."; + default: + throw new ArgumentException("Unknown type"); + } + } + } +} diff --git a/src/csharp/Grpc.Core/Profiling/ProfilerScope.cs b/src/csharp/Grpc.Core/Profiling/ProfilerScope.cs new file mode 100644 index 00000000000..413f3a1a358 --- /dev/null +++ b/src/csharp/Grpc.Core/Profiling/ProfilerScope.cs @@ -0,0 +1,60 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; +using System.IO; +using System.Threading; +using Grpc.Core.Internal; + +namespace Grpc.Core.Profiling +{ + // Allows declaring Begin and End of a profiler scope with a using statement. + // declared as struct for better performance. + internal struct ProfilerScope : IDisposable + { + readonly IProfiler profiler; + readonly string tag; + + public ProfilerScope(IProfiler profiler, string tag) + { + this.profiler = profiler; + this.tag = tag; + this.profiler.Begin(this.tag); + } + + public void Dispose() + { + profiler.End(tag); + } + } +} diff --git a/src/csharp/Grpc.Core/Profiling/Profilers.cs b/src/csharp/Grpc.Core/Profiling/Profilers.cs new file mode 100644 index 00000000000..c8123347f2b --- /dev/null +++ b/src/csharp/Grpc.Core/Profiling/Profilers.cs @@ -0,0 +1,131 @@ +#region Copyright notice and license + +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#endregion + +using System; +using System.IO; +using System.Threading; +using Grpc.Core.Internal; + +namespace Grpc.Core.Profiling +{ + internal static class Profilers + { + static readonly NopProfiler defaultProfiler = new NopProfiler(); + static readonly ThreadLocal profilers = new ThreadLocal(); + + public static IProfiler ForCurrentThread() + { + return profilers.Value ?? defaultProfiler; + } + + public static void SetForCurrentThread(IProfiler profiler) + { + profilers.Value = profiler; + } + + public static ProfilerScope NewScope(this IProfiler profiler, string tag) + { + return new ProfilerScope(profiler, tag); + } + } + + internal class NopProfiler : IProfiler + { + public void Begin(string tag) + { + } + + public void End(string tag) + { + } + + public void Mark(string tag) + { + } + } + + // Profiler using Timespec.PreciseNow + internal class BasicProfiler : IProfiler + { + ProfilerEntry[] entries; + int count; + + public BasicProfiler() : this(1024*1024) + { + } + + public BasicProfiler(int capacity) + { + this.entries = new ProfilerEntry[capacity]; + } + + public void Begin(string tag) { + AddEntry(new ProfilerEntry(Timespec.PreciseNow, ProfilerEntry.Type.BEGIN, tag)); + } + + public void End(string tag) { + AddEntry(new ProfilerEntry(Timespec.PreciseNow, ProfilerEntry.Type.END, tag)); + } + + public void Mark(string tag) { + AddEntry(new ProfilerEntry(Timespec.PreciseNow, ProfilerEntry.Type.MARK, tag)); + } + + public void Reset() + { + count = 0; + } + + public void Dump(string filepath) + { + using (var stream = new StreamWriter(filepath)) + { + Dump(stream); + } + } + + public void Dump(TextWriter stream) + { + for (int i = 0; i < count; i++) + { + var entry = entries[i]; + stream.WriteLine(entry.ToString()); + } + } + + // NOT THREADSAFE! + void AddEntry(ProfilerEntry entry) { + entries[count++] = entry; + } + } +} From 2271ab5aeab775cb0a6382ae1e0a6979e7e4210c Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Wed, 28 Oct 2015 13:59:14 -0700 Subject: [PATCH 3/6] Create a separate utility for reporting. --- tools/run_tests/dockerjob.py | 2 +- tools/run_tests/generate_reports.py | 172 +++++++++++++++++++++++++++ tools/run_tests/jobset.py | 29 ++--- tools/run_tests/run_interop_tests.py | 136 ++------------------- tools/run_tests/run_tests.py | 11 +- 5 files changed, 190 insertions(+), 160 deletions(-) create mode 100644 tools/run_tests/generate_reports.py diff --git a/tools/run_tests/dockerjob.py b/tools/run_tests/dockerjob.py index 1d67fe3033e..7d64222ba0b 100755 --- a/tools/run_tests/dockerjob.py +++ b/tools/run_tests/dockerjob.py @@ -101,7 +101,7 @@ class DockerJob: def __init__(self, spec): self._spec = spec - self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={}, xml_report=None) + self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={}) self._container_name = spec.container_name def mapped_port(self, port): diff --git a/tools/run_tests/generate_reports.py b/tools/run_tests/generate_reports.py new file mode 100644 index 00000000000..6ba47e9c2ef --- /dev/null +++ b/tools/run_tests/generate_reports.py @@ -0,0 +1,172 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Generate XML and HTML test reports.""" + +import os +import xml.etree.cElementTree as ET + + +def render_xml_report(resultset, xml_report): + """Generate JUnit-like XML report.""" + root = ET.Element('testsuites') + testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', + name='tests') + for shortname, results in resultset.iteritems(): + for result in results: + xml_test = ET.SubElement(testsuite, 'testcase', name=shortname) + if result.elapsed_time: + xml_test.set('time', str(result.elapsed_time)) + ET.SubElement(xml_test, 'system-out').text = result.message + if result.state == 'FAILED': + ET.SubElement(xml_test, 'failure', message='Failure') + elif result.state == 'TIMEOUT': + ET.SubElement(xml_test, 'error', message='Timeout') + tree = ET.ElementTree(root) + tree.write(xml_report, encoding='UTF-8') + + +# TODO(adelez): Use mako template. +def fill_one_test_result(shortname, resultset, html_str): + if shortname in resultset: + # Because interop tests does not have runs_per_test flag, each test is run + # once. So there should only be one element for each result. + result = resultset[shortname][0] + if result.state == 'PASSED': + html_str = '%sPASS\n' % html_str + else: + tooltip = '' + if result.returncode > 0 or result.message: + if result.returncode > 0: + tooltip = 'returncode: %d ' % result.returncode + if result.message: + escaped_msg = result.message.replace('"', '"') + tooltip = '%smessage: %s' % (tooltip, escaped_msg) + if result.state == 'FAILED': + html_str = '%s' % html_str + if tooltip: + html_str = ('%sFAIL\n' % + (html_str, tooltip)) + else: + html_str = '%sFAIL\n' % html_str + elif result.state == 'TIMEOUT': + html_str = '%s' % html_str + if tooltip: + html_str = ('%sTIMEOUT\n' + % (html_str, tooltip)) + else: + html_str = '%sTIMEOUT\n' % html_str + else: + html_str = '%sNot implemented\n' % html_str + + return html_str + + +def render_html_report(client_langs, server_langs, test_cases, auth_test_cases, + resultset, num_failures, cloud_to_prod): + """Generate html report.""" + sorted_test_cases = sorted(test_cases) + sorted_auth_test_cases = sorted(auth_test_cases) + sorted_client_langs = sorted(client_langs) + sorted_server_langs = sorted(server_langs) + html_str = ('\n' + '\n' + 'Interop Test Result\n' + '\n') + if num_failures > 1: + html_str = ( + '%s

%d tests failed!

\n' % + (html_str, num_failures)) + elif num_failures: + html_str = ( + '%s

%d test failed!

\n' % + (html_str, num_failures)) + else: + html_str = ( + '%s

All tests passed!

\n' % + html_str) + if cloud_to_prod: + # Each column header is the client language. + html_str = ('%s

Cloud to Prod

\n' + '\n' + '\n' + '\n') % html_str + for client_lang in sorted_client_langs: + html_str = '%s\n' % html_str + for test_case in sorted_test_cases + sorted_auth_test_cases: + html_str = '%s\n' % (html_str, test_case) + for client_lang in sorted_client_langs: + if not test_case in sorted_auth_test_cases: + shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case) + else: + shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case) + html_str = fill_one_test_result(shortname, resultset, html_str) + html_str = '%s\n' % html_str + html_str = '%s
Client languages ►%s\n' % (html_str, client_lang) + html_str = '%s
%s
\n' % html_str + if server_langs: + for test_case in sorted_test_cases: + # Each column header is the client language. + html_str = ('%s

%s

\n' + '\n' + '\n' + '\n') % (html_str, test_case) + for client_lang in sorted_client_langs: + html_str = '%s\n' % html_str + # Each row head is the server language. + for server_lang in sorted_server_langs: + html_str = '%s\n' % (html_str, server_lang) + # Fill up the cells with test result. + for client_lang in sorted_client_langs: + shortname = 'cloud_to_cloud:%s:%s_server:%s' % ( + client_lang, server_lang, test_case) + html_str = fill_one_test_result(shortname, resultset, html_str) + html_str = '%s\n' % html_str + html_str = '%s
Client languages ►
' + 'Server languages ▼		%s\n' % (html_str, client_lang) + html_str = '%s
%s
\n' % html_str + + html_str = ('%s\n' + '\n' + '\n' + '') % html_str + + # Write to reports/index.html as set up in Jenkins plugin. + html_report_dir = 'reports' + if not os.path.exists(html_report_dir): + os.mkdir(html_report_dir) + html_file_path = os.path.join(html_report_dir, 'index.html') + with open(html_file_path, 'w') as f: + f.write(html_str) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index a8ff9f613fb..95bd7c7256a 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -39,7 +39,6 @@ import subprocess import sys import tempfile import time -import xml.etree.cElementTree as ET _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count() @@ -190,14 +189,12 @@ class JobResult(object): class Job(object): """Manages one job.""" - def __init__(self, spec, bin_hash, newline_on_success, travis, add_env, xml_report): + def __init__(self, spec, bin_hash, newline_on_success, travis, add_env): self._spec = spec self._bin_hash = bin_hash self._newline_on_success = newline_on_success self._travis = travis self._add_env = add_env.copy() - self._xml_test = ET.SubElement(xml_report, 'testcase', - name=self._spec.shortname) if xml_report is not None else None self._retries = 0 self._timeout_retries = 0 self._suppress_failure_message = False @@ -229,15 +226,12 @@ class Job(object): self._tempfile.seek(0) stdout = self._tempfile.read() filtered_stdout = _filter_stdout(stdout) - # TODO: looks like jenkins master is slow because parsing the junit results XMLs is not - # implemented efficiently. This is an experiment to workaround the issue by making sure - # results.xml file is small enough. + # TODO: looks like jenkins master is slow because parsing the junit + # results XMLs is not implemented efficiently. This is an experiment to + # workaround the issue by making sure results.xml file is small enough. filtered_stdout = filtered_stdout[-128:] self.result.message = filtered_stdout self.result.elapsed_time = elapsed - if self._xml_test is not None: - self._xml_test.set('time', str(elapsed)) - ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout if self._process.returncode != 0: if self._retries < self._spec.flake_retries: message('FLAKE', '%s [ret=%d, pid=%d]' % ( @@ -256,8 +250,6 @@ class Job(object): self.result.state = 'FAILED' self.result.num_failures += 1 self.result.returncode = self._process.returncode - if self._xml_test is not None: - ET.SubElement(self._xml_test, 'failure', message='Failure') else: self._state = _SUCCESS message('PASSED', '%s [time=%.1fsec; retries=%d;%d]' % ( @@ -285,9 +277,6 @@ class Job(object): self.kill() self.result.state = 'TIMEOUT' self.result.num_failures += 1 - if self._xml_test is not None: - ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout - ET.SubElement(self._xml_test, 'error', message='Timeout') return self._state def kill(self): @@ -305,7 +294,7 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, add_env, cache, xml_report): + stop_on_failure, add_env, cache): self._running = set() self._check_cancelled = check_cancelled self._cancelled = False @@ -317,7 +306,6 @@ class Jobset(object): self._cache = cache self._stop_on_failure = stop_on_failure self._hashes = {} - self._xml_report = xml_report self._add_env = add_env self.resultset = {} @@ -349,8 +337,7 @@ class Jobset(object): bin_hash, self._newline_on_success, self._travis, - self._add_env, - self._xml_report) + self._add_env) self._running.add(job) self.resultset[job.GetSpec().shortname] = [] return True @@ -424,13 +411,11 @@ def run(cmdlines, infinite_runs=False, stop_on_failure=False, cache=None, - xml_report=None, add_env={}): js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, newline_on_success, travis, stop_on_failure, add_env, - cache if cache is not None else NoCache(), - xml_report) + cache if cache is not None else NoCache()) for cmdline in cmdlines: if not js.start(cmdline): break diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 729f962bb18..efbfe1f269a 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -33,7 +33,7 @@ import argparse import dockerjob import itertools -import xml.etree.cElementTree as ET +import generate_reports import jobset import multiprocessing import os @@ -471,126 +471,6 @@ def build_interop_image_jobspec(language, tag=None): return build_job -# TODO(adelez): Use mako template. -def fill_one_test_result(shortname, resultset, html_str): - if shortname in resultset: - # Because interop tests does not have runs_per_test flag, each test is run - # once. So there should only be one element for each result. - result = resultset[shortname][0] - if result.state == 'PASSED': - html_str = '%sPASS\n' % html_str - else: - tooltip = '' - if result.returncode > 0 or result.message: - if result.returncode > 0: - tooltip = 'returncode: %d ' % result.returncode - if result.message: - escaped_msg = result.message.replace('"', '"') - tooltip = '%smessage: %s' % (tooltip, escaped_msg) - if result.state == 'FAILED': - html_str = '%s' % html_str - if tooltip: - html_str = ('%sFAIL\n' % - (html_str, tooltip)) - else: - html_str = '%sFAIL\n' % html_str - elif result.state == 'TIMEOUT': - html_str = '%s' % html_str - if tooltip: - html_str = ('%sTIMEOUT\n' - % (html_str, tooltip)) - else: - html_str = '%sTIMEOUT\n' % html_str - else: - html_str = '%sNot implemented\n' % html_str - - return html_str - - -def render_html_report(client_langs, server_langs, resultset, - num_failures): - """Generate html report.""" - sorted_test_cases = sorted(_TEST_CASES) - sorted_auth_test_cases = sorted(_AUTH_TEST_CASES) - sorted_client_langs = sorted(client_langs) - sorted_server_langs = sorted(server_langs) - html_str = ('\n' - '\n' - 'Interop Test Result\n' - '\n') - if num_failures > 1: - html_str = ( - '%s

%d tests failed!

\n' % - (html_str, num_failures)) - elif num_failures: - html_str = ( - '%s

%d test failed!

\n' % - (html_str, num_failures)) - else: - html_str = ( - '%s

All tests passed!

\n' % - html_str) - if args.cloud_to_prod_auth or args.cloud_to_prod: - # Each column header is the client language. - html_str = ('%s

Cloud to Prod

\n' - '\n' - '\n' - '\n') % html_str - for client_lang in sorted_client_langs: - html_str = '%s\n' % html_str - for test_case in sorted_test_cases + sorted_auth_test_cases: - html_str = '%s\n' % (html_str, test_case) - for client_lang in sorted_client_langs: - if not test_case in sorted_auth_test_cases: - shortname = 'cloud_to_prod:%s:%s' % (client_lang, test_case) - else: - shortname = 'cloud_to_prod_auth:%s:%s' % (client_lang, test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - html_str = '%s\n' % html_str - html_str = '%s
Client languages ►%s\n' % (html_str, client_lang) - html_str = '%s
%s
\n' % html_str - if servers: - for test_case in sorted_test_cases: - # Each column header is the client language. - html_str = ('%s

%s

\n' - '\n' - '\n' - '\n') % (html_str, test_case) - for client_lang in sorted_client_langs: - html_str = '%s\n' % html_str - # Each row head is the server language. - for server_lang in sorted_server_langs: - html_str = '%s\n' % (html_str, server_lang) - # Fill up the cells with test result. - for client_lang in sorted_client_langs: - shortname = 'cloud_to_cloud:%s:%s_server:%s' % ( - client_lang, server_lang, test_case) - html_str = fill_one_test_result(shortname, resultset, html_str) - html_str = '%s\n' % html_str - html_str = '%s
Client languages ►
' - 'Server languages ▼		%s\n' % (html_str, client_lang) - html_str = '%s
%s
\n' % html_str - - html_str = ('%s\n' - '\n' - '\n' - '') % html_str - - # Write to reports/index.html as set up in Jenkins plugin. - html_report_dir = 'reports' - if not os.path.exists(html_report_dir): - os.mkdir(html_report_dir) - html_file_path = os.path.join(html_report_dir, 'index.html') - with open(html_file_path, 'w') as f: - f.write(html_str) - - argp = argparse.ArgumentParser(description='Run interop tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(_LANGUAGES), @@ -740,22 +620,20 @@ try: dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) - root = ET.Element('testsuites') - testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') - num_failures, resultset = jobset.run(jobs, newline_on_success=True, - maxjobs=args.jobs, xml_report=testsuite) + maxjobs=args.jobs) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) - tree = ET.ElementTree(root) - tree.write('report.xml', encoding='UTF-8') + # Generate XML report. + generate_reports.render_xml_report(resultset, 'report.xml') # Generate HTML report. - render_html_report(set([str(l) for l in languages]), servers, - resultset, num_failures) + generate_reports.render_html_report( + set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, + resultset, num_failures, args.cloud_to_prod_auth or args.cloud_to_prod) finally: # Check if servers are still running. diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4232637c7f8..725b4812399 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -46,9 +46,9 @@ import sys import tempfile import traceback import time -import xml.etree.cElementTree as ET import urllib2 +import generate_reports import jobset import watch_dirs @@ -867,15 +867,11 @@ def _build_and_run( else itertools.repeat(massaged_one_run, runs_per_test)) all_runs = itertools.chain.from_iterable(runs_sequence) - root = ET.Element('testsuites') if xml_report else None - testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') if xml_report else None - number_failures, resultset = jobset.run( - all_runs, check_cancelled, newline_on_success=newline_on_success, + all_runs, check_cancelled, newline_on_success=newline_on_success, travis=travis, infinite_runs=infinite_runs, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, cache=cache if not xml_report else None, - xml_report=testsuite, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) if resultset: for k, v in resultset.iteritems(): @@ -894,8 +890,7 @@ def _build_and_run( for antagonist in antagonists: antagonist.kill() if xml_report: - tree = ET.ElementTree(root) - tree.write(xml_report, encoding='UTF-8') + generate_reports.render_xml_report(resultset, xml_report) number_failures, _ = jobset.run( post_tests_steps, maxjobs=1, stop_on_failure=True, From a30f829e62ca245db7d2ddcd22054b1494ce3a4c Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Mon, 2 Nov 2015 13:15:46 -0800 Subject: [PATCH 4/6] Renamed report_utils.py. --- tools/run_tests/{generate_reports.py => report_utils.py} | 0 tools/run_tests/run_interop_tests.py | 8 +++----- tools/run_tests/run_tests.py | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) rename tools/run_tests/{generate_reports.py => report_utils.py} (100%) diff --git a/tools/run_tests/generate_reports.py b/tools/run_tests/report_utils.py similarity index 100% rename from tools/run_tests/generate_reports.py rename to tools/run_tests/report_utils.py diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index efbfe1f269a..cebe2468867 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -33,10 +33,10 @@ import argparse import dockerjob import itertools -import generate_reports import jobset import multiprocessing import os +import report_utils import subprocess import sys import tempfile @@ -627,11 +627,9 @@ try: else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) - # Generate XML report. - generate_reports.render_xml_report(resultset, 'report.xml') + report_utils.render_xml_report(resultset, 'report.xml') - # Generate HTML report. - generate_reports.render_html_report( + report_utils.render_html_report( set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES, resultset, num_failures, args.cloud_to_prod_auth or args.cloud_to_prod) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 725b4812399..ae7899e47ee 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -48,8 +48,8 @@ import traceback import time import urllib2 -import generate_reports import jobset +import report_utils import watch_dirs ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) @@ -890,7 +890,7 @@ def _build_and_run( for antagonist in antagonists: antagonist.kill() if xml_report: - generate_reports.render_xml_report(resultset, xml_report) + report_utils.render_xml_report(resultset, xml_report) number_failures, _ = jobset.run( post_tests_steps, maxjobs=1, stop_on_failure=True, From 472bb6849cd9b51589e27d0d124e6a883d1de1e6 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Tue, 3 Nov 2015 19:09:01 +0100 Subject: [PATCH 5/6] Fixing proto dependencies for targets that aren't libraries. --- Makefile | 5 +++++ templates/Makefile.template | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Makefile b/Makefile index a25523310ea..819ac7a6494 100644 --- a/Makefile +++ b/Makefile @@ -10235,6 +10235,7 @@ ifneq ($(NO_DEPS),true) -include $(RECONNECT_INTEROP_CLIENT_OBJS:.o=.dep) endif endif +$(OBJDIR)/$(CONFIG)/test/cpp/interop/reconnect_interop_client.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc RECONNECT_INTEROP_SERVER_SRC = \ @@ -10281,6 +10282,7 @@ ifneq ($(NO_DEPS),true) -include $(RECONNECT_INTEROP_SERVER_OBJS:.o=.dep) endif endif +$(OBJDIR)/$(CONFIG)/test/cpp/interop/reconnect_interop_server.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc SECURE_AUTH_CONTEXT_TEST_SRC = \ @@ -10571,6 +10573,9 @@ ifneq ($(NO_DEPS),true) -include $(STRESS_TEST_OBJS:.o=.dep) endif endif +$(OBJDIR)/$(CONFIG)/test/cpp/interop/interop_client.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_interop_client.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_test.o: $(GENDIR)/test/proto/empty.pb.cc $(GENDIR)/test/proto/empty.grpc.pb.cc $(GENDIR)/test/proto/messages.pb.cc $(GENDIR)/test/proto/messages.grpc.pb.cc $(GENDIR)/test/proto/test.pb.cc $(GENDIR)/test/proto/test.grpc.pb.cc SYNC_STREAMING_PING_PONG_TEST_SRC = \ diff --git a/templates/Makefile.template b/templates/Makefile.template index 115d8136b53..20d14c797f9 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -1851,6 +1851,11 @@ endif % endif % endif + % for src in tgt.src: + % if not proto_re.match(src) and any(proto_re.match(src2) for src2 in tgt.src): + $(OBJDIR)/$(CONFIG)/${os.path.splitext(src)[0]}.o: ${' '.join(proto_to_cc(src2) for src2 in tgt.src if proto_re.match(src2))} + % endif + % endfor ifneq ($(OPENSSL_DEP),) From d01cbe324c81a492b14062ff5884f38cb2c34c4f Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Mon, 2 Nov 2015 14:20:43 -0800 Subject: [PATCH 6/6] Move string filter to report_utils --- tools/run_tests/jobset.py | 25 ++++--------------------- tools/run_tests/report_utils.py | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 95bd7c7256a..0c4d1b8143c 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -34,7 +34,6 @@ import multiprocessing import os import platform import signal -import string import subprocess import sys import tempfile @@ -42,6 +41,7 @@ import time _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count() +_MAX_RESULT_SIZE = 8192 # setup a signal handler so that signal.pause registers 'something' @@ -129,14 +129,6 @@ def which(filename): raise Exception('%s not found' % filename) -def _filter_stdout(stdout): - """Filters out nonprintable and XML-illegal characters from stdout.""" - # keep whitespaces but remove formfeed and vertical tab characters - # that make XML report unparseable. - return filter(lambda x: x in string.printable and x != '\f' and x != '\v', - stdout.decode(errors='ignore')) - - class JobSpec(object): """Specifies what to run for a job.""" @@ -221,16 +213,11 @@ class Job(object): def state(self, update_cache): """Poll current state of the job. Prints messages at completion.""" + self._tempfile.seek(0) + stdout = self._tempfile.read() + self.result.message = stdout[-_MAX_RESULT_SIZE:] if self._state == _RUNNING and self._process.poll() is not None: elapsed = time.time() - self._start - self._tempfile.seek(0) - stdout = self._tempfile.read() - filtered_stdout = _filter_stdout(stdout) - # TODO: looks like jenkins master is slow because parsing the junit - # results XMLs is not implemented efficiently. This is an experiment to - # workaround the issue by making sure results.xml file is small enough. - filtered_stdout = filtered_stdout[-128:] - self.result.message = filtered_stdout self.result.elapsed_time = elapsed if self._process.returncode != 0: if self._retries < self._spec.flake_retries: @@ -259,10 +246,6 @@ class Job(object): if self._bin_hash: update_cache.finished(self._spec.identity(), self._bin_hash) elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds: - self._tempfile.seek(0) - stdout = self._tempfile.read() - filtered_stdout = _filter_stdout(stdout) - self.result.message = filtered_stdout if self._timeout_retries < self._spec.timeout_retries: message('TIMEOUT_FLAKE', self._spec.shortname, stdout, do_newline=True) self._timeout_retries += 1 diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py index 6ba47e9c2ef..57a93d0da05 100644 --- a/tools/run_tests/report_utils.py +++ b/tools/run_tests/report_utils.py @@ -30,9 +30,25 @@ """Generate XML and HTML test reports.""" import os +import string import xml.etree.cElementTree as ET +def _filter_msg(msg, output_format): + """Filters out nonprintable and illegal characters from the message.""" + if output_format in ['XML', 'HTML']: + # keep whitespaces but remove formfeed and vertical tab characters + # that make XML report unparseable. + filtered_msg = filter( + lambda x: x in string.printable and x != '\f' and x != '\v', + msg.decode(errors='ignore')) + if output_format == 'HTML': + filtered_msg = filtered_msg.replace('"', '"') + return filtered_msg + else: + return msg + + def render_xml_report(resultset, xml_report): """Generate JUnit-like XML report.""" root = ET.Element('testsuites') @@ -43,7 +59,8 @@ def render_xml_report(resultset, xml_report): xml_test = ET.SubElement(testsuite, 'testcase', name=shortname) if result.elapsed_time: xml_test.set('time', str(result.elapsed_time)) - ET.SubElement(xml_test, 'system-out').text = result.message + ET.SubElement(xml_test, 'system-out').text = _filter_msg(result.message, + 'XML') if result.state == 'FAILED': ET.SubElement(xml_test, 'failure', message='Failure') elif result.state == 'TIMEOUT': @@ -66,7 +83,7 @@ def fill_one_test_result(shortname, resultset, html_str): if result.returncode > 0: tooltip = 'returncode: %d ' % result.returncode if result.message: - escaped_msg = result.message.replace('"', '"') + escaped_msg = _filter_msg(result.message, 'HTML') tooltip = '%smessage: %s' % (tooltip, escaped_msg) if result.state == 'FAILED': html_str = '%s' % html_str