Refactored benchmark reporting mechanism.

It now allows pluggging in "reporter" instances to process the benchmark results arbitrarily.
This would allow, for example, to send results to a leaderboard and/or other systems for tracking performance metrics.

test/cpp/qps/async_streaming_ping_pong_test.cc

@@ -31,6 +31,8 @@
  *
  */
 
+#include <set>
+
 #include <grpc/support/log.h>
 
 #include <signal.h>
@@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
 static void RunAsyncStreamingPingPong() {
   gpr_log(GPR_INFO, "Running Async Streaming Ping Pong");
 
+  ReportersRegistry reporters_registry;
+  reporters_registry.Register(new GprLogReporter("LogReporter"));
+
   ClientConfig client_config;
   client_config.set_client_type(ASYNC_CLIENT);
   client_config.set_enable_ssl(false);
@@ -64,8 +69,10 @@ static void RunAsyncStreamingPingPong() {
   const auto result =
       RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
 
-  ReportQPS(result);
+  std::set<ReportType> types;
-  ReportLatency(result);
+  types.insert(grpc::testing::ReportType::REPORT_QPS);
+  types.insert(grpc::testing::ReportType::REPORT_LATENCY);
+  reporters_registry.Report({client_config, server_config, result}, types);
 }
 
 }  // namespace testing

test/cpp/qps/async_unary_ping_pong_test.cc

@@ -31,6 +31,8 @@
  *
  */
 
+#include <set>
+
 #include <grpc/support/log.h>
 
 #include <signal.h>
@@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
 static void RunAsyncUnaryPingPong() {
   gpr_log(GPR_INFO, "Running Async Unary Ping Pong");
 
+  ReportersRegistry reporters_registry;
+  reporters_registry.Register(new GprLogReporter("LogReporter"));
+
   ClientConfig client_config;
   client_config.set_client_type(ASYNC_CLIENT);
   client_config.set_enable_ssl(false);
@@ -64,8 +69,10 @@ static void RunAsyncUnaryPingPong() {
   const auto result =
       RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
 
-  ReportQPS(result);
+  std::set<ReportType> types;
-  ReportLatency(result);
+  types.insert(grpc::testing::ReportType::REPORT_QPS);
+  types.insert(grpc::testing::ReportType::REPORT_LATENCY);
+  reporters_registry.Report({client_config, server_config, result}, types);
 }
 
 }  // namespace testing

test/cpp/qps/qps_driver.cc

@@ -31,6 +31,8 @@
  *
  */
 
+#include <set>
+
 #include <gflags/gflags.h>
 #include <grpc/support/log.h>
 
@@ -67,10 +69,17 @@ using grpc::testing::ClientType;
 using grpc::testing::ServerType;
 using grpc::testing::RpcType;
 using grpc::testing::ResourceUsage;
+using grpc::testing::ReportersRegistry;
+using grpc::testing::GprLogReporter;
+using grpc::testing::ReportData;
+using grpc::testing::ReportType;
 
 int main(int argc, char** argv) {
   grpc::testing::InitTest(&argc, &argv, true);
 
+  ReportersRegistry reporters_registry;
+  reporters_registry.Register(new GprLogReporter("LogReporter"));
+
   RpcType rpc_type;
   GPR_ASSERT(RpcType_Parse(FLAGS_rpc_type, &rpc_type));
 
@@ -103,14 +112,13 @@ int main(int argc, char** argv) {
                FLAGS_server_threads <  FLAGS_client_channels *
                FLAGS_outstanding_rpcs_per_channel));
 
-  auto result = RunScenario(client_config, FLAGS_num_clients,
+  const auto result = RunScenario(
-                            server_config, FLAGS_num_servers,
+      client_config, FLAGS_num_clients, server_config, FLAGS_num_servers,
-                            FLAGS_warmup_seconds, FLAGS_benchmark_seconds,
+      FLAGS_warmup_seconds, FLAGS_benchmark_seconds, FLAGS_local_workers);
-                            FLAGS_local_workers);
 
-  ReportQPSPerCore(result, server_config);
+  std::set<ReportType> types;
-  ReportLatency(result);
+  types.insert(grpc::testing::ReportType::REPORT_ALL);
-  ReportTimes(result);
+  reporters_registry.Report({client_config, server_config, result}, types);
 
   return 0;
 }

test/cpp/qps/qps_test.cc

@@ -31,6 +31,8 @@
  *
  */
 
+#include <set>
+
 #include <grpc/support/log.h>
 
 #include <signal.h>
@@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
 static void RunQPS() {
   gpr_log(GPR_INFO, "Running QPS test");
 
+  ReportersRegistry reporters_registry;
+  reporters_registry.Register(new GprLogReporter("LogReporter"));
+
   ClientConfig client_config;
   client_config.set_client_type(ASYNC_CLIENT);
   client_config.set_enable_ssl(false);
@@ -64,8 +69,11 @@ static void RunQPS() {
   const auto result =
       RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
 
-  ReportQPSPerCore(result, server_config);
+  std::set<ReportType> types;
-  ReportLatency(result);
+  types.insert(grpc::testing::ReportType::REPORT_QPS_PER_CORE);
+  types.insert(grpc::testing::ReportType::REPORT_LATENCY);
+  reporters_registry.Report({client_config, server_config, result}, types);
+
 }
 
 }  // namespace testing

test/cpp/qps/report.cc

@@ -39,27 +39,73 @@
 namespace grpc {
 namespace testing {
 
-// QPS: XXX
+// ReporterRegistry implementation.
-void ReportQPS(const ScenarioResult& result) {
+void ReportersRegistry::Register(const Reporter* reporter) {
+  reporters_.emplace_back(reporter);
+}
+
+std::vector<string> ReportersRegistry::GetNamesRegistered() const {
+  std::vector<string> names;
+  for (const auto& reporter : reporters_) {
+    names.push_back(reporter->name());
+  }
+  return names;
+}
+
+void ReportersRegistry::Report(const ReportData& data,
+                               const std::set<ReportType>& types) const {
+  for (const auto& reporter : reporters_) {
+    reporter->Report(data, types);
+  }
+}
+
+// Reporter implementation.
+void Reporter::Report(const ReportData& data,
+                      const std::set<ReportType>& types) const {
+  for (ReportType rtype : types) {
+    bool all = false;
+    switch (rtype) {
+      case REPORT_ALL:
+        all = true;
+      case REPORT_QPS:
+        ReportQPS(data.scenario_result);
+        if (!all) break;
+      case REPORT_QPS_PER_CORE:
+        ReportQPSPerCore(data.scenario_result, data.server_config);
+        if (!all) break;
+      case REPORT_LATENCY:
+        ReportLatency(data.scenario_result);
+        if (!all) break;
+      case REPORT_TIMES:
+        ReportTimes(data.scenario_result);
+        if (!all) break;
+    }
+    if (all) break;
+  }
+}
+
+// GprLogReporter implementation.
+void GprLogReporter::ReportQPS(const ScenarioResult& result) const {
   gpr_log(GPR_INFO, "QPS: %.1f",
           result.latencies.Count() /
               average(result.client_resources,
                       [](ResourceUsage u) { return u.wall_time; }));
 }
 
-// QPS: XXX (YYY/server core)
+void GprLogReporter::ReportQPSPerCore(const ScenarioResult& result,
-void ReportQPSPerCore(const ScenarioResult& result, const ServerConfig& server_config) {
+                                      const ServerConfig& server_config) const {
   auto qps =
       result.latencies.Count() /
       average(result.client_resources,
           [](ResourceUsage u) { return u.wall_time; });
 
-  gpr_log(GPR_INFO, "QPS: %.1f (%.1f/server core)", qps, qps/server_config.threads());
+  gpr_log(GPR_INFO, "QPS: %.1f (%.1f/server core)", qps,
+          qps / server_config.threads());
 }
 
-// Latency (50/90/95/99/99.9%-ile): AA/BB/CC/DD/EE us
+void GprLogReporter::ReportLatency(const ScenarioResult& result) const {
-void ReportLatency(const ScenarioResult& result) {
+  gpr_log(GPR_INFO,
-  gpr_log(GPR_INFO, "Latencies (50/90/95/99/99.9%%-ile): %.1f/%.1f/%.1f/%.1f/%.1f us",
+          "Latencies (50/90/95/99/99.9%%-ile): %.1f/%.1f/%.1f/%.1f/%.1f us",
           result.latencies.Percentile(50) / 1000,
           result.latencies.Percentile(90) / 1000,
           result.latencies.Percentile(95) / 1000,
@@ -67,7 +113,7 @@ void ReportLatency(const ScenarioResult& result) {
           result.latencies.Percentile(99.9) / 1000);
 }
 
-void ReportTimes(const ScenarioResult& result) {
+void GprLogReporter::ReportTimes(const ScenarioResult& result) const {
   gpr_log(GPR_INFO, "Server system time: %.2f%%",
           100.0 * sum(result.server_resources,
                       [](ResourceUsage u) { return u.system_time; }) /

test/cpp/qps/report.h

@@ -34,22 +34,112 @@
 #ifndef TEST_QPS_REPORT_H
 #define TEST_QPS_REPORT_H
 
+#include <memory>
+#include <set>
+#include <vector>
+#include <grpc++/config.h>
+
 #include "test/cpp/qps/driver.h"
+#include "test/cpp/qps/qpstest.grpc.pb.h"
 
 namespace grpc {
 namespace testing {
 
-// QPS: XXX
+/** General set of data required for report generation. */
-void ReportQPS(const ScenarioResult& result);
+struct ReportData {
-// QPS: XXX (YYY/server core)
+  const ClientConfig& client_config;
-void ReportQPSPerCore(const ScenarioResult& result, const ServerConfig& config);
+  const ServerConfig& server_config;
-// Latency (50/90/95/99/99.9%-ile): AA/BB/CC/DD/EE us
+  const ScenarioResult& scenario_result;
-void ReportLatency(const ScenarioResult& result);
+};
-// Server system time: XX%
+
-// Server user time: XX%
+/** Specifies the type of performance report we are interested in.
-// Client system time: XX%
+ *
-// Client user time: XX%
+ *  \note The special type \c REPORT_ALL is equivalent to specifying all the
-void ReportTimes(const ScenarioResult& result);
+ *  other fields. */
+enum ReportType {
+  /** Equivalent to the combination of all other fields. */
+  REPORT_ALL,
+  /** Report only QPS information. */
+  REPORT_QPS,
+  /** Report only QPS per core information. */
+  REPORT_QPS_PER_CORE,
+  /** Report latency info for the 50, 90, 95, 99 and 99.9th percentiles. */
+  REPORT_LATENCY,
+  /** Report user and system time. */
+  REPORT_TIMES
+};
+
+class Reporter;
+
+/** A registry of Reporter instances.
+ *
+ * Instances registered will be taken into account by the Report() method.
+ */
+class ReportersRegistry {
+ public:
+  /** Adds the \c reporter to the registry.
+   * \attention Takes ownership of \c reporter. */
+  void Register(const Reporter* reporter);
+
+  /** Returns the names of the registered \c Reporter instances. */
+  std::vector<string> GetNamesRegistered() const;
+
+  /** Triggers the reporting for all registered \c Reporter instances.
+   *
+   * \param data Configuration and results for the scenario being reported.
+   * \param types A collection of report types to include in the report. */
+  void Report(const ReportData& data,
+              const std::set<ReportType>& types) const;
+
+ private:
+  std::vector<std::unique_ptr<const Reporter> > reporters_;
+};
+
+/** Interface for all reporters. */
+class Reporter {
+ public:
+  /** Construct a reporter with the given \a name. */
+  Reporter(const string& name) : name_(name) {}
+
+  /** Returns this reporter's name.
+   *
+   * Names are constants, set at construction time. */
+  string name() const { return name_; }
+
+  /** Template method responsible for the generation of the requested types. */
+  void Report(const ReportData& data, const std::set<ReportType>& types) const;
+
+ protected:
+  /** Reports QPS for the given \a result. */
+  virtual void ReportQPS(const ScenarioResult& result) const = 0;
+  /** Reports QPS per core as (YYY/server core). */
+  virtual void ReportQPSPerCore(const ScenarioResult& result,
+                        const ServerConfig& config) const = 0;
+  /** Reports latencies for the 50, 90, 95, 99 and 99.9 percentiles, in ms. */
+  virtual void ReportLatency(const ScenarioResult& result) const = 0;
+
+  /** Reports system and user time for client and server systems. */
+  virtual void ReportTimes(const ScenarioResult& result) const = 0;
+
+ private:
+  const string name_;
+};
+
+
+// Reporters.
+
+/** Reporter to gpr_log(GPR_INFO). */
+class GprLogReporter : public Reporter {
+ public:
+  GprLogReporter(const string& name) : Reporter(name) {}
+
+ private:
+  void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE;
+  void ReportQPSPerCore(const ScenarioResult& result,
+                        const ServerConfig& config) const GRPC_OVERRIDE;
+  void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE;
+  void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE;
+};
 
 }  // namespace testing
 }  // namespace grpc

test/cpp/qps/sync_streaming_ping_pong_test.cc

@@ -31,6 +31,8 @@
  *
  */
 
+#include <set>
+
 #include <grpc/support/log.h>
 
 #include <signal.h>
@@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
 static void RunSynchronousStreamingPingPong() {
   gpr_log(GPR_INFO, "Running Synchronous Streaming Ping Pong");
 
+  ReportersRegistry reporters_registry;
+  reporters_registry.Register(new GprLogReporter("LogReporter"));
+
   ClientConfig client_config;
   client_config.set_client_type(SYNCHRONOUS_CLIENT);
   client_config.set_enable_ssl(false);
@@ -63,8 +68,11 @@ static void RunSynchronousStreamingPingPong() {
   const auto result =
       RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
 
-  ReportQPS(result);
+  std::set<ReportType> types;
-  ReportLatency(result);
+  types.insert(grpc::testing::ReportType::REPORT_QPS);
+  types.insert(grpc::testing::ReportType::REPORT_LATENCY);
+  reporters_registry.Report({client_config, server_config, result}, types);
+
 }
 
 }  // namespace testing

test/cpp/qps/sync_unary_ping_pong_test.cc

@@ -31,6 +31,8 @@
  *
  */
 
+#include <set>
+
 #include <grpc/support/log.h>
 
 #include <signal.h>
@@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
 static void RunSynchronousUnaryPingPong() {
   gpr_log(GPR_INFO, "Running Synchronous Unary Ping Pong");
 
+  ReportersRegistry reporters_registry;
+  reporters_registry.Register(new GprLogReporter("LogReporter"));
+
   ClientConfig client_config;
   client_config.set_client_type(SYNCHRONOUS_CLIENT);
   client_config.set_enable_ssl(false);
@@ -63,8 +68,10 @@ static void RunSynchronousUnaryPingPong() {
   const auto result =
       RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
 
-  ReportQPS(result);
+  std::set<ReportType> types;
-  ReportLatency(result);
+  types.insert(grpc::testing::ReportType::REPORT_QPS);
+  types.insert(grpc::testing::ReportType::REPORT_LATENCY);
+  reporters_registry.Report({client_config, server_config, result}, types);
 }
 
 }  // namespace testing