From e18ed03c043d237b81bc32ddbd7d79593e2f7f19 Mon Sep 17 00:00:00 2001
From: Esun Kim <veblush@google.com>
Date: Thu, 2 May 2019 16:58:37 -0700
Subject: [PATCH] Made gRPC inialized after entering main function in
 microbenchmarks.

---
 test/cpp/microbenchmarks/bm_alarm.cc            |  3 +--
 test/cpp/microbenchmarks/bm_byte_buffer.cc      |  4 +---
 test/cpp/microbenchmarks/bm_call_create.cc      |  3 +--
 test/cpp/microbenchmarks/bm_channel.cc          |  3 +--
 test/cpp/microbenchmarks/bm_chttp2_hpack.cc     |  3 +--
 test/cpp/microbenchmarks/bm_chttp2_transport.cc |  6 +++---
 test/cpp/microbenchmarks/bm_closure.cc          |  3 +--
 test/cpp/microbenchmarks/bm_cq.cc               |  3 +--
 test/cpp/microbenchmarks/bm_error.cc            |  3 +--
 .../bm_fullstack_streaming_ping_pong.cc         |  4 +---
 .../bm_fullstack_streaming_pump.cc              |  4 +---
 .../cpp/microbenchmarks/bm_fullstack_trickle.cc |  8 +++-----
 .../bm_fullstack_unary_ping_pong.cc             |  4 +---
 test/cpp/microbenchmarks/bm_metadata.cc         |  3 +--
 test/cpp/microbenchmarks/bm_pollset.cc          |  3 +--
 test/cpp/microbenchmarks/bm_timer.cc            |  3 +--
 test/cpp/microbenchmarks/fullstack_fixtures.h   |  4 ++--
 test/cpp/microbenchmarks/helpers.cc             | 17 ++++++++++++++++-
 test/cpp/microbenchmarks/helpers.h              | 14 +++++---------
 19 files changed, 43 insertions(+), 52 deletions(-)

diff --git a/test/cpp/microbenchmarks/bm_alarm.cc b/test/cpp/microbenchmarks/bm_alarm.cc
index 64aad6476de..d95771a57c6 100644
--- a/test/cpp/microbenchmarks/bm_alarm.cc
+++ b/test/cpp/microbenchmarks/bm_alarm.cc
@@ -30,8 +30,6 @@
 namespace grpc {
 namespace testing {
 
-auto& force_library_initialization = Library::get();
-
 static void BM_Alarm_Tag_Immediate(benchmark::State& state) {
   TrackCounters track_counters;
   CompletionQueue cq;
@@ -57,6 +55,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_byte_buffer.cc b/test/cpp/microbenchmarks/bm_byte_buffer.cc
index 644c27c4873..595cc734b69 100644
--- a/test/cpp/microbenchmarks/bm_byte_buffer.cc
+++ b/test/cpp/microbenchmarks/bm_byte_buffer.cc
@@ -30,7 +30,6 @@ namespace grpc {
 namespace testing {
 
 static void BM_ByteBuffer_Copy(benchmark::State& state) {
-  Library::get();
   int num_slices = state.range(0);
   size_t slice_size = state.range(1);
   std::vector<grpc::Slice> slices;
@@ -48,7 +47,6 @@ static void BM_ByteBuffer_Copy(benchmark::State& state) {
 BENCHMARK(BM_ByteBuffer_Copy)->Ranges({{1, 64}, {1, 1024 * 1024}});
 
 static void BM_ByteBufferReader_Next(benchmark::State& state) {
-  Library::get();
   const int num_slices = state.range(0);
   constexpr size_t kSliceSize = 16;
   std::vector<grpc_slice> slices;
@@ -82,7 +80,6 @@ static void BM_ByteBufferReader_Next(benchmark::State& state) {
 BENCHMARK(BM_ByteBufferReader_Next)->Ranges({{64 * 1024, 1024 * 1024}});
 
 static void BM_ByteBufferReader_Peek(benchmark::State& state) {
-  Library::get();
   const int num_slices = state.range(0);
   constexpr size_t kSliceSize = 16;
   std::vector<grpc_slice> slices;
@@ -125,6 +122,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc
index 53087a551b5..3bd1464b2aa 100644
--- a/test/cpp/microbenchmarks/bm_call_create.cc
+++ b/test/cpp/microbenchmarks/bm_call_create.cc
@@ -47,8 +47,6 @@
 #include "test/cpp/microbenchmarks/helpers.h"
 #include "test/cpp/util/test_config.h"
 
-auto& force_library_initialization = Library::get();
-
 void BM_Zalloc(benchmark::State& state) {
   // speed of light for call creation is zalloc, so benchmark a few interesting
   // sizes
@@ -823,6 +821,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_channel.cc b/test/cpp/microbenchmarks/bm_channel.cc
index 15ac9975400..88856c3439b 100644
--- a/test/cpp/microbenchmarks/bm_channel.cc
+++ b/test/cpp/microbenchmarks/bm_channel.cc
@@ -23,8 +23,6 @@
 #include "test/cpp/microbenchmarks/helpers.h"
 #include "test/cpp/util/test_config.h"
 
-auto& force_library_initialization = Library::get();
-
 class ChannelDestroyerFixture {
  public:
   ChannelDestroyerFixture() {}
@@ -83,6 +81,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc
index 0521166db95..d0ed1da7671 100644
--- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc
+++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc
@@ -36,8 +36,6 @@
 #include "test/cpp/microbenchmarks/helpers.h"
 #include "test/cpp/util/test_config.h"
 
-auto& force_library_initialization = Library::get();
-
 static grpc_slice MakeSlice(std::vector<uint8_t> bytes) {
   grpc_slice s = grpc_slice_malloc(bytes.size());
   uint8_t* p = GRPC_SLICE_START_PTR(s);
@@ -928,6 +926,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_chttp2_transport.cc b/test/cpp/microbenchmarks/bm_chttp2_transport.cc
index dc10fcb89d3..05504584c20 100644
--- a/test/cpp/microbenchmarks/bm_chttp2_transport.cc
+++ b/test/cpp/microbenchmarks/bm_chttp2_transport.cc
@@ -36,8 +36,6 @@
 #include "test/cpp/microbenchmarks/helpers.h"
 #include "test/cpp/util/test_config.h"
 
-auto& force_library_initialization = Library::get();
-
 ////////////////////////////////////////////////////////////////////////////////
 // Helper classes
 //
@@ -57,7 +55,8 @@ class DummyEndpoint : public grpc_endpoint {
                                                    get_fd,
                                                    can_track_err};
     grpc_endpoint::vtable = &my_vtable;
-    ru_ = grpc_resource_user_create(Library::get().rq(), "dummy_endpoint");
+    ru_ = grpc_resource_user_create(LibraryInitializer::get().rq(),
+                                    "dummy_endpoint");
   }
 
   void PushInput(grpc_slice slice) {
@@ -642,6 +641,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_closure.cc b/test/cpp/microbenchmarks/bm_closure.cc
index e1f1e92d4d8..84b1c536bf0 100644
--- a/test/cpp/microbenchmarks/bm_closure.cc
+++ b/test/cpp/microbenchmarks/bm_closure.cc
@@ -30,8 +30,6 @@
 #include "test/cpp/microbenchmarks/helpers.h"
 #include "test/cpp/util/test_config.h"
 
-auto& force_library_initialization = Library::get();
-
 static void BM_NoOpExecCtx(benchmark::State& state) {
   TrackCounters track_counters;
   while (state.KeepRunning()) {
@@ -425,6 +423,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_cq.cc b/test/cpp/microbenchmarks/bm_cq.cc
index a7cb939265f..263314deb93 100644
--- a/test/cpp/microbenchmarks/bm_cq.cc
+++ b/test/cpp/microbenchmarks/bm_cq.cc
@@ -32,8 +32,6 @@
 namespace grpc {
 namespace testing {
 
-auto& force_library_initialization = Library::get();
-
 static void BM_CreateDestroyCpp(benchmark::State& state) {
   TrackCounters track_counters;
   while (state.KeepRunning()) {
@@ -156,6 +154,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_error.cc b/test/cpp/microbenchmarks/bm_error.cc
index ae557a580aa..a71817f7e54 100644
--- a/test/cpp/microbenchmarks/bm_error.cc
+++ b/test/cpp/microbenchmarks/bm_error.cc
@@ -27,8 +27,6 @@
 #include "test/cpp/microbenchmarks/helpers.h"
 #include "test/cpp/util/test_config.h"
 
-auto& force_library_initialization = Library::get();
-
 class ErrorDeleter {
  public:
   void operator()(grpc_error* error) { GRPC_ERROR_UNREF(error); }
@@ -318,6 +316,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
index 34df77aca3c..60ca9a093ed 100644
--- a/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
+++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc
@@ -24,9 +24,6 @@
 namespace grpc {
 namespace testing {
 
-// force library initialization
-auto& force_library_initialization = Library::get();
-
 /*******************************************************************************
  * CONFIGURATIONS
  */
@@ -122,6 +119,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc
index da98f3cbcd4..d4533e6c78e 100644
--- a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc
+++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc
@@ -28,9 +28,6 @@ namespace testing {
  * CONFIGURATIONS
  */
 
-// force library initialization
-auto& force_library_initialization = Library::get();
-
 BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, TCP)
     ->Range(0, 128 * 1024 * 1024);
 BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS)
@@ -72,6 +69,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc
index 1af92d2c80c..2d733bcaa42 100644
--- a/test/cpp/microbenchmarks/bm_fullstack_trickle.cc
+++ b/test/cpp/microbenchmarks/bm_fullstack_trickle.cc
@@ -214,8 +214,8 @@ class TrickledCHTTP2 : public EndpointPairFixture {
   static grpc_endpoint_pair MakeEndpoints(size_t kilobits,
                                           grpc_passthru_endpoint_stats* stats) {
     grpc_endpoint_pair p;
-    grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(),
-                                  stats);
+    grpc_passthru_endpoint_create(&p.client, &p.server,
+                                  LibraryInitializer::get().rq(), stats);
     double bytes_per_second = 125.0 * kilobits;
     p.client = grpc_trickle_endpoint_create(p.client, bytes_per_second);
     p.server = grpc_trickle_endpoint_create(p.server, bytes_per_second);
@@ -235,9 +235,6 @@ class TrickledCHTTP2 : public EndpointPairFixture {
   }
 };
 
-// force library initialization
-auto& force_library_initialization = Library::get();
-
 static void TrickleCQNext(TrickledCHTTP2* fixture, void** t, bool* ok,
                           int64_t iteration) {
   while (true) {
@@ -465,6 +462,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   grpc_timer_manager_set_threading(false);
diff --git a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc
index d4bd58b9838..3e8f9344021 100644
--- a/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc
+++ b/test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc
@@ -24,9 +24,6 @@
 namespace grpc {
 namespace testing {
 
-// force library initialization
-auto& force_library_initialization = Library::get();
-
 /*******************************************************************************
  * CONFIGURATIONS
  */
@@ -174,6 +171,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_metadata.cc b/test/cpp/microbenchmarks/bm_metadata.cc
index 553b33c4028..ff8fe541dfc 100644
--- a/test/cpp/microbenchmarks/bm_metadata.cc
+++ b/test/cpp/microbenchmarks/bm_metadata.cc
@@ -27,8 +27,6 @@
 #include "test/cpp/microbenchmarks/helpers.h"
 #include "test/cpp/util/test_config.h"
 
-auto& force_library_initialization = Library::get();
-
 static void BM_SliceFromStatic(benchmark::State& state) {
   TrackCounters track_counters;
   while (state.KeepRunning()) {
@@ -298,6 +296,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_pollset.cc b/test/cpp/microbenchmarks/bm_pollset.cc
index 050c7f7c171..f8e36f178e4 100644
--- a/test/cpp/microbenchmarks/bm_pollset.cc
+++ b/test/cpp/microbenchmarks/bm_pollset.cc
@@ -40,8 +40,6 @@
 #include <unistd.h>
 #endif
 
-auto& force_library_initialization = Library::get();
-
 static void shutdown_ps(void* ps, grpc_error* error) {
   grpc_pollset_destroy(static_cast<grpc_pollset*>(ps));
 }
@@ -264,6 +262,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/bm_timer.cc b/test/cpp/microbenchmarks/bm_timer.cc
index f5a411251b5..53aaead992d 100644
--- a/test/cpp/microbenchmarks/bm_timer.cc
+++ b/test/cpp/microbenchmarks/bm_timer.cc
@@ -32,8 +32,6 @@
 namespace grpc {
 namespace testing {
 
-auto& force_library_initialization = Library::get();
-
 struct TimerClosure {
   grpc_timer timer;
   grpc_closure closure;
@@ -111,6 +109,7 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
 }  // namespace benchmark
 
 int main(int argc, char** argv) {
+  LibraryInitializer libInit;
   ::benchmark::Initialize(&argc, argv);
   ::grpc::testing::InitTest(&argc, &argv, false);
   benchmark::RunTheBenchmarksNamespaced();
diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h
index 80ccab3e6ef..075a09c63eb 100644
--- a/test/cpp/microbenchmarks/fullstack_fixtures.h
+++ b/test/cpp/microbenchmarks/fullstack_fixtures.h
@@ -299,8 +299,8 @@ class InProcessCHTTP2WithExplicitStats : public EndpointPairFixture {
 
   static grpc_endpoint_pair MakeEndpoints(grpc_passthru_endpoint_stats* stats) {
     grpc_endpoint_pair p;
-    grpc_passthru_endpoint_create(&p.client, &p.server, Library::get().rq(),
-                                  stats);
+    grpc_passthru_endpoint_create(&p.client, &p.server,
+                                  LibraryInitializer::get().rq(), stats);
     return p;
   }
 };
diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc
index d4070de7481..7d78e21aef7 100644
--- a/test/cpp/microbenchmarks/helpers.cc
+++ b/test/cpp/microbenchmarks/helpers.cc
@@ -21,8 +21,12 @@
 #include "test/cpp/microbenchmarks/helpers.h"
 
 static grpc::internal::GrpcLibraryInitializer g_gli_initializer;
+static LibraryInitializer* g_libraryInitializer;
+
+LibraryInitializer::LibraryInitializer() {
+  GPR_ASSERT(g_libraryInitializer == nullptr);
+  g_libraryInitializer = this;
 
-Library::Library() {
   g_gli_initializer.summon();
 #ifdef GPR_LOW_LEVEL_COUNTERS
   grpc_memory_counters_init();
@@ -31,6 +35,17 @@ Library::Library() {
   rq_ = grpc_resource_quota_create("bm");
 }
 
+LibraryInitializer::~LibraryInitializer() {
+  g_libraryInitializer = nullptr;
+  init_lib_.shutdown();
+  grpc_resource_quota_unref(rq_);
+}
+
+LibraryInitializer& LibraryInitializer::get() {
+  GPR_ASSERT(g_libraryInitializer != nullptr);
+  return *g_libraryInitializer;
+}
+
 void TrackCounters::Finish(benchmark::State& state) {
   std::ostringstream out;
   for (const auto& l : labels_) {
diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h
index 770966aa189..b712c85d354 100644
--- a/test/cpp/microbenchmarks/helpers.h
+++ b/test/cpp/microbenchmarks/helpers.h
@@ -29,20 +29,16 @@
 #include <benchmark/benchmark.h>
 #include <grpcpp/impl/grpc_library.h>
 
-class Library {
+class LibraryInitializer {
  public:
-  static Library& get() {
-    static Library lib;
-    return lib;
-  }
+  LibraryInitializer();
+  ~LibraryInitializer();
 
   grpc_resource_quota* rq() { return rq_; }
 
- private:
-  Library();
-
-  ~Library() { init_lib_.shutdown(); }
+  static LibraryInitializer& get();
 
+ private:
   grpc::internal::GrpcLibrary init_lib_;
   grpc_resource_quota* rq_;
 };