From efe9fd0d60e49bb6ec8f01f73879b9c2d020ddf9 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 26 Jul 2019 17:08:45 +0200 Subject: [PATCH 1/4] scalable benchmarks --- .../CommonThreadedBase.cs | 71 +++++++++++++++++-- .../ScalabityExampleBenchmark.cs | 56 +++++++++++++++ 2 files changed, 123 insertions(+), 4 deletions(-) create mode 100644 src/csharp/Grpc.Microbenchmarks/ScalabityExampleBenchmark.cs diff --git a/src/csharp/Grpc.Microbenchmarks/CommonThreadedBase.cs b/src/csharp/Grpc.Microbenchmarks/CommonThreadedBase.cs index c6bdf471b8a..c42c6481ffb 100644 --- a/src/csharp/Grpc.Microbenchmarks/CommonThreadedBase.cs +++ b/src/csharp/Grpc.Microbenchmarks/CommonThreadedBase.cs @@ -17,6 +17,8 @@ #endregion using System; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; @@ -35,22 +37,42 @@ namespace Grpc.Microbenchmarks { protected virtual bool NeedsEnvironment => true; - [Params(1, 2, 4, 8, 12)] + [Params(1, 2, 4, 6)] public int ThreadCount { get; set; } protected GrpcEnvironment Environment { get; private set; } + private List workers; + + private List> dispatchQueues; + [GlobalSetup] public virtual void Setup() { - ThreadPool.GetMinThreads(out var workers, out var iocp); - if (workers <= ThreadCount) ThreadPool.SetMinThreads(ThreadCount + 1, iocp); + dispatchQueues = new List>(); + workers = new List(); + for (int i = 0; i < ThreadCount; i++) + { + var dispatchQueue = new BlockingCollection(); + var thread = new Thread(new ThreadStart(() => WorkerThreadBody(dispatchQueue))); + thread.Name = string.Format("threaded benchmark worker {0}", i); + thread.Start(); + workers.Add(thread); + dispatchQueues.Add(dispatchQueue); + } + if (NeedsEnvironment) Environment = GrpcEnvironment.AddRef(); } [GlobalCleanup] public virtual void Cleanup() { + for (int i = 0; i < ThreadCount; i++) + { + dispatchQueues[i].Add(null); // null action request termination of the worker thread. + workers[i].Join(); + } + if (Environment != null) { Environment = null; @@ -58,9 +80,50 @@ namespace Grpc.Microbenchmarks } } + /// + /// Runs the operation in parallel (once on each worker thread). + /// This method tries to incur as little + /// overhead as possible, but there is some inherent overhead + /// that is hard to avoid (thread hop etc.). Therefore it is strongly + /// recommended that the benchmarked operation runs long enough to + /// make this overhead negligible. + /// protected void RunConcurrent(Action operation) { - Parallel.For(0, ThreadCount, _ => operation()); + var workItemTasks = new Task[ThreadCount]; + for (int i = 0; i < ThreadCount; i++) + { + var tcs = new TaskCompletionSource(); + var workItem = new Action(() => + { + try + { + operation(); + tcs.SetResult(null); + } + catch (Exception e) + { + tcs.SetException(e); + } + }); + workItemTasks[i] = tcs.Task; + dispatchQueues[i].Add(workItem); + } + Task.WaitAll(workItemTasks); + } + + private void WorkerThreadBody(BlockingCollection dispatchQueue) + { + while(true) + { + var workItem = dispatchQueue.Take(); + if (workItem == null) + { + // stop the worker if null action was provided + break; + } + workItem(); + } } } } diff --git a/src/csharp/Grpc.Microbenchmarks/ScalabityExampleBenchmark.cs b/src/csharp/Grpc.Microbenchmarks/ScalabityExampleBenchmark.cs new file mode 100644 index 00000000000..1ef117c83e7 --- /dev/null +++ b/src/csharp/Grpc.Microbenchmarks/ScalabityExampleBenchmark.cs @@ -0,0 +1,56 @@ +#region Copyright notice and license + +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.Runtime.InteropServices; +using BenchmarkDotNet.Attributes; +using Grpc.Core.Internal; + +namespace Grpc.Microbenchmarks +{ + public class ScalabilityExampleBenchmark : CommonThreadedBase + { + protected override bool NeedsEnvironment => false; + + // An example of testing scalability of a method that scales perfectly. + // This method provides a baseline for how well can CommonThreadedBase + // measure scalability. + const int Iterations = 50 * 1000 * 1000; // High number to make the overhead of RunConcurrent negligible. + [Benchmark(OperationsPerInvoke = Iterations)] + public void PerfectScalingExample() + { + RunConcurrent(() => { RunBody(); }); + } + + private int RunBody() + { + int result = 0; + for (int i = 0; i < Iterations; i++) + { + // perform some operation that is completely independent from + // other threads and therefore should scale perfectly if given + // a dedicated thread. + for (int j = 0; j < 100; j++) + { + result = result ^ i ^ j ; + } + } + return result; + } + } +} From 7372c195e717e65c1a60e7b07165d0eaf89b7509 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 1 Aug 2019 04:00:23 +0200 Subject: [PATCH 2/4] adjust iteration count in scalability benchmarks --- src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs | 2 +- src/csharp/Grpc.Microbenchmarks/PInvokeByteArrayBenchmark.cs | 2 +- src/csharp/Grpc.Microbenchmarks/SendMessageBenchmark.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs b/src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs index e4cd7eeb1f6..2374148bfe0 100644 --- a/src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs +++ b/src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs @@ -27,7 +27,7 @@ namespace Grpc.Microbenchmarks [Params(false, true)] public bool UseSharedRegistry { get; set; } - const int Iterations = 1000000; // High number to make the overhead of RunConcurrent negligible. + const int Iterations = 5 * 1000 * 1000; // High number to make the overhead of RunConcurrent negligible. [Benchmark(OperationsPerInvoke = Iterations)] public void RegisterExtract() { diff --git a/src/csharp/Grpc.Microbenchmarks/PInvokeByteArrayBenchmark.cs b/src/csharp/Grpc.Microbenchmarks/PInvokeByteArrayBenchmark.cs index de4e635580f..1ede7453db6 100644 --- a/src/csharp/Grpc.Microbenchmarks/PInvokeByteArrayBenchmark.cs +++ b/src/csharp/Grpc.Microbenchmarks/PInvokeByteArrayBenchmark.cs @@ -32,7 +32,7 @@ namespace Grpc.Microbenchmarks [Params(0)] public int PayloadSize { get; set; } - const int Iterations = 1000000; // High number to make the overhead of RunConcurrent negligible. + const int Iterations = 5 * 1000 * 1000; // High number to make the overhead of RunConcurrent negligible. [Benchmark(OperationsPerInvoke = Iterations)] public void AllocFree() { diff --git a/src/csharp/Grpc.Microbenchmarks/SendMessageBenchmark.cs b/src/csharp/Grpc.Microbenchmarks/SendMessageBenchmark.cs index d9554db00db..8496813efb0 100644 --- a/src/csharp/Grpc.Microbenchmarks/SendMessageBenchmark.cs +++ b/src/csharp/Grpc.Microbenchmarks/SendMessageBenchmark.cs @@ -36,7 +36,7 @@ namespace Grpc.Microbenchmarks [Params(0)] public int PayloadSize { get; set; } - const int Iterations = 1000000; // High number to make the overhead of RunConcurrent negligible. + const int Iterations = 5 * 1000 * 1000; // High number to make the overhead of RunConcurrent negligible. [Benchmark(OperationsPerInvoke = Iterations)] public void SendMessage() { From a16a43945889870438cd6217574861e1276aff04 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 1 Aug 2019 04:53:35 +0200 Subject: [PATCH 3/4] fix CompletionRegistryBenchmark for UseSharedRegistry=true --- .../Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs b/src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs index 2374148bfe0..9426882b54b 100644 --- a/src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs +++ b/src/csharp/Grpc.Microbenchmarks/CompletionRegistryBenchmark.cs @@ -31,8 +31,9 @@ namespace Grpc.Microbenchmarks [Benchmark(OperationsPerInvoke = Iterations)] public void RegisterExtract() { - RunConcurrent(() => { - CompletionRegistry sharedRegistry = UseSharedRegistry ? new CompletionRegistry(Environment, () => BatchContextSafeHandle.Create(), () => RequestCallContextSafeHandle.Create()) : null; + CompletionRegistry sharedRegistry = UseSharedRegistry ? new CompletionRegistry(Environment, () => BatchContextSafeHandle.Create(), () => RequestCallContextSafeHandle.Create()) : null; + RunConcurrent(() => + { RunBody(sharedRegistry); }); } From 46253995bb5ac760f20e47bb88a2aeb809d47a76 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 1 Aug 2019 04:55:04 +0200 Subject: [PATCH 4/4] add AtomicCounterBenchmark --- .../AtomicCounterBenchmark.cs | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/csharp/Grpc.Microbenchmarks/AtomicCounterBenchmark.cs diff --git a/src/csharp/Grpc.Microbenchmarks/AtomicCounterBenchmark.cs b/src/csharp/Grpc.Microbenchmarks/AtomicCounterBenchmark.cs new file mode 100644 index 00000000000..03984f40728 --- /dev/null +++ b/src/csharp/Grpc.Microbenchmarks/AtomicCounterBenchmark.cs @@ -0,0 +1,52 @@ +#region Copyright notice and license + +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.Runtime.InteropServices; +using BenchmarkDotNet.Attributes; +using Grpc.Core.Internal; + +namespace Grpc.Microbenchmarks +{ + public class AtomicCounterBenchmark : CommonThreadedBase + { + protected override bool NeedsEnvironment => false; + + // An example of testing scalability of a method that scales perfectly. + // This method provides a baseline for how well can CommonThreadedBase + // measure scalability. + const int Iterations = 20 * 1000 * 1000; // High number to make the overhead of RunConcurrent negligible. + [Benchmark(OperationsPerInvoke = Iterations)] + + public void SharedAtomicCounterIncrement() + { + RunConcurrent(() => { RunBody(); }); + } + + readonly AtomicCounter sharedCounter = new AtomicCounter(); + + private int RunBody() + { + for (int i = 0; i < Iterations; i++) + { + sharedCounter.Increment(); + } + return (int) sharedCounter.Count; + } + } +}