Refactored benchmark reporting mechanism.

It now allows pluggging in "reporter" instances to process the benchmark results arbitrarily.
This would allow, for example, to send results to a leaderboard and/or other systems for tracking performance metrics.
pull/1649/head
David Garcia Quintas 10 years ago
parent faff1caf4d
commit cdbdedbf23
  1. 11
      test/cpp/qps/async_streaming_ping_pong_test.cc
  2. 11
      test/cpp/qps/async_unary_ping_pong_test.cc
  3. 22
      test/cpp/qps/qps_driver.cc
  4. 12
      test/cpp/qps/qps_test.cc
  5. 66
      test/cpp/qps/report.cc
  6. 112
      test/cpp/qps/report.h
  7. 12
      test/cpp/qps/sync_streaming_ping_pong_test.cc
  8. 11
      test/cpp/qps/sync_unary_ping_pong_test.cc

@ -31,6 +31,8 @@
*
*/
#include <set>
#include <grpc/support/log.h>
#include <signal.h>
@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
static void RunAsyncStreamingPingPong() {
gpr_log(GPR_INFO, "Running Async Streaming Ping Pong");
ReportersRegistry reporters_registry;
reporters_registry.Register(new GprLogReporter("LogReporter"));
ClientConfig client_config;
client_config.set_client_type(ASYNC_CLIENT);
client_config.set_enable_ssl(false);
@ -64,8 +69,10 @@ static void RunAsyncStreamingPingPong() {
const auto result =
RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
ReportQPS(result);
ReportLatency(result);
std::set<ReportType> types;
types.insert(grpc::testing::ReportType::REPORT_QPS);
types.insert(grpc::testing::ReportType::REPORT_LATENCY);
reporters_registry.Report({client_config, server_config, result}, types);
}
} // namespace testing

@ -31,6 +31,8 @@
*
*/
#include <set>
#include <grpc/support/log.h>
#include <signal.h>
@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
static void RunAsyncUnaryPingPong() {
gpr_log(GPR_INFO, "Running Async Unary Ping Pong");
ReportersRegistry reporters_registry;
reporters_registry.Register(new GprLogReporter("LogReporter"));
ClientConfig client_config;
client_config.set_client_type(ASYNC_CLIENT);
client_config.set_enable_ssl(false);
@ -64,8 +69,10 @@ static void RunAsyncUnaryPingPong() {
const auto result =
RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
ReportQPS(result);
ReportLatency(result);
std::set<ReportType> types;
types.insert(grpc::testing::ReportType::REPORT_QPS);
types.insert(grpc::testing::ReportType::REPORT_LATENCY);
reporters_registry.Report({client_config, server_config, result}, types);
}
} // namespace testing

@ -31,6 +31,8 @@
*
*/
#include <set>
#include <gflags/gflags.h>
#include <grpc/support/log.h>
@ -67,10 +69,17 @@ using grpc::testing::ClientType;
using grpc::testing::ServerType;
using grpc::testing::RpcType;
using grpc::testing::ResourceUsage;
using grpc::testing::ReportersRegistry;
using grpc::testing::GprLogReporter;
using grpc::testing::ReportData;
using grpc::testing::ReportType;
int main(int argc, char** argv) {
grpc::testing::InitTest(&argc, &argv, true);
ReportersRegistry reporters_registry;
reporters_registry.Register(new GprLogReporter("LogReporter"));
RpcType rpc_type;
GPR_ASSERT(RpcType_Parse(FLAGS_rpc_type, &rpc_type));
@ -103,14 +112,13 @@ int main(int argc, char** argv) {
FLAGS_server_threads < FLAGS_client_channels *
FLAGS_outstanding_rpcs_per_channel));
auto result = RunScenario(client_config, FLAGS_num_clients,
server_config, FLAGS_num_servers,
FLAGS_warmup_seconds, FLAGS_benchmark_seconds,
FLAGS_local_workers);
const auto result = RunScenario(
client_config, FLAGS_num_clients, server_config, FLAGS_num_servers,
FLAGS_warmup_seconds, FLAGS_benchmark_seconds, FLAGS_local_workers);
ReportQPSPerCore(result, server_config);
ReportLatency(result);
ReportTimes(result);
std::set<ReportType> types;
types.insert(grpc::testing::ReportType::REPORT_ALL);
reporters_registry.Report({client_config, server_config, result}, types);
return 0;
}

@ -31,6 +31,8 @@
*
*/
#include <set>
#include <grpc/support/log.h>
#include <signal.h>
@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
static void RunQPS() {
gpr_log(GPR_INFO, "Running QPS test");
ReportersRegistry reporters_registry;
reporters_registry.Register(new GprLogReporter("LogReporter"));
ClientConfig client_config;
client_config.set_client_type(ASYNC_CLIENT);
client_config.set_enable_ssl(false);
@ -64,8 +69,11 @@ static void RunQPS() {
const auto result =
RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
ReportQPSPerCore(result, server_config);
ReportLatency(result);
std::set<ReportType> types;
types.insert(grpc::testing::ReportType::REPORT_QPS_PER_CORE);
types.insert(grpc::testing::ReportType::REPORT_LATENCY);
reporters_registry.Report({client_config, server_config, result}, types);
}
} // namespace testing

@ -39,27 +39,73 @@
namespace grpc {
namespace testing {
// QPS: XXX
void ReportQPS(const ScenarioResult& result) {
// ReporterRegistry implementation.
void ReportersRegistry::Register(const Reporter* reporter) {
reporters_.emplace_back(reporter);
}
std::vector<string> ReportersRegistry::GetNamesRegistered() const {
std::vector<string> names;
for (const auto& reporter : reporters_) {
names.push_back(reporter->name());
}
return names;
}
void ReportersRegistry::Report(const ReportData& data,
const std::set<ReportType>& types) const {
for (const auto& reporter : reporters_) {
reporter->Report(data, types);
}
}
// Reporter implementation.
void Reporter::Report(const ReportData& data,
const std::set<ReportType>& types) const {
for (ReportType rtype : types) {
bool all = false;
switch (rtype) {
case REPORT_ALL:
all = true;
case REPORT_QPS:
ReportQPS(data.scenario_result);
if (!all) break;
case REPORT_QPS_PER_CORE:
ReportQPSPerCore(data.scenario_result, data.server_config);
if (!all) break;
case REPORT_LATENCY:
ReportLatency(data.scenario_result);
if (!all) break;
case REPORT_TIMES:
ReportTimes(data.scenario_result);
if (!all) break;
}
if (all) break;
}
}
// GprLogReporter implementation.
void GprLogReporter::ReportQPS(const ScenarioResult& result) const {
gpr_log(GPR_INFO, "QPS: %.1f",
result.latencies.Count() /
average(result.client_resources,
[](ResourceUsage u) { return u.wall_time; }));
}
// QPS: XXX (YYY/server core)
void ReportQPSPerCore(const ScenarioResult& result, const ServerConfig& server_config) {
auto qps =
void GprLogReporter::ReportQPSPerCore(const ScenarioResult& result,
const ServerConfig& server_config) const {
auto qps =
result.latencies.Count() /
average(result.client_resources,
[](ResourceUsage u) { return u.wall_time; });
gpr_log(GPR_INFO, "QPS: %.1f (%.1f/server core)", qps, qps/server_config.threads());
gpr_log(GPR_INFO, "QPS: %.1f (%.1f/server core)", qps,
qps / server_config.threads());
}
// Latency (50/90/95/99/99.9%-ile): AA/BB/CC/DD/EE us
void ReportLatency(const ScenarioResult& result) {
gpr_log(GPR_INFO, "Latencies (50/90/95/99/99.9%%-ile): %.1f/%.1f/%.1f/%.1f/%.1f us",
void GprLogReporter::ReportLatency(const ScenarioResult& result) const {
gpr_log(GPR_INFO,
"Latencies (50/90/95/99/99.9%%-ile): %.1f/%.1f/%.1f/%.1f/%.1f us",
result.latencies.Percentile(50) / 1000,
result.latencies.Percentile(90) / 1000,
result.latencies.Percentile(95) / 1000,
@ -67,7 +113,7 @@ void ReportLatency(const ScenarioResult& result) {
result.latencies.Percentile(99.9) / 1000);
}
void ReportTimes(const ScenarioResult& result) {
void GprLogReporter::ReportTimes(const ScenarioResult& result) const {
gpr_log(GPR_INFO, "Server system time: %.2f%%",
100.0 * sum(result.server_resources,
[](ResourceUsage u) { return u.system_time; }) /

@ -34,22 +34,112 @@
#ifndef TEST_QPS_REPORT_H
#define TEST_QPS_REPORT_H
#include <memory>
#include <set>
#include <vector>
#include <grpc++/config.h>
#include "test/cpp/qps/driver.h"
#include "test/cpp/qps/qpstest.grpc.pb.h"
namespace grpc {
namespace testing {
// QPS: XXX
void ReportQPS(const ScenarioResult& result);
// QPS: XXX (YYY/server core)
void ReportQPSPerCore(const ScenarioResult& result, const ServerConfig& config);
// Latency (50/90/95/99/99.9%-ile): AA/BB/CC/DD/EE us
void ReportLatency(const ScenarioResult& result);
// Server system time: XX%
// Server user time: XX%
// Client system time: XX%
// Client user time: XX%
void ReportTimes(const ScenarioResult& result);
/** General set of data required for report generation. */
struct ReportData {
const ClientConfig& client_config;
const ServerConfig& server_config;
const ScenarioResult& scenario_result;
};
/** Specifies the type of performance report we are interested in.
*
* \note The special type \c REPORT_ALL is equivalent to specifying all the
* other fields. */
enum ReportType {
/** Equivalent to the combination of all other fields. */
REPORT_ALL,
/** Report only QPS information. */
REPORT_QPS,
/** Report only QPS per core information. */
REPORT_QPS_PER_CORE,
/** Report latency info for the 50, 90, 95, 99 and 99.9th percentiles. */
REPORT_LATENCY,
/** Report user and system time. */
REPORT_TIMES
};
class Reporter;
/** A registry of Reporter instances.
*
* Instances registered will be taken into account by the Report() method.
*/
class ReportersRegistry {
public:
/** Adds the \c reporter to the registry.
* \attention Takes ownership of \c reporter. */
void Register(const Reporter* reporter);
/** Returns the names of the registered \c Reporter instances. */
std::vector<string> GetNamesRegistered() const;
/** Triggers the reporting for all registered \c Reporter instances.
*
* \param data Configuration and results for the scenario being reported.
* \param types A collection of report types to include in the report. */
void Report(const ReportData& data,
const std::set<ReportType>& types) const;
private:
std::vector<std::unique_ptr<const Reporter> > reporters_;
};
/** Interface for all reporters. */
class Reporter {
public:
/** Construct a reporter with the given \a name. */
Reporter(const string& name) : name_(name) {}
/** Returns this reporter's name.
*
* Names are constants, set at construction time. */
string name() const { return name_; }
/** Template method responsible for the generation of the requested types. */
void Report(const ReportData& data, const std::set<ReportType>& types) const;
protected:
/** Reports QPS for the given \a result. */
virtual void ReportQPS(const ScenarioResult& result) const = 0;
/** Reports QPS per core as (YYY/server core). */
virtual void ReportQPSPerCore(const ScenarioResult& result,
const ServerConfig& config) const = 0;
/** Reports latencies for the 50, 90, 95, 99 and 99.9 percentiles, in ms. */
virtual void ReportLatency(const ScenarioResult& result) const = 0;
/** Reports system and user time for client and server systems. */
virtual void ReportTimes(const ScenarioResult& result) const = 0;
private:
const string name_;
};
// Reporters.
/** Reporter to gpr_log(GPR_INFO). */
class GprLogReporter : public Reporter {
public:
GprLogReporter(const string& name) : Reporter(name) {}
private:
void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE;
void ReportQPSPerCore(const ScenarioResult& result,
const ServerConfig& config) const GRPC_OVERRIDE;
void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE;
void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE;
};
} // namespace testing
} // namespace grpc

@ -31,6 +31,8 @@
*
*/
#include <set>
#include <grpc/support/log.h>
#include <signal.h>
@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
static void RunSynchronousStreamingPingPong() {
gpr_log(GPR_INFO, "Running Synchronous Streaming Ping Pong");
ReportersRegistry reporters_registry;
reporters_registry.Register(new GprLogReporter("LogReporter"));
ClientConfig client_config;
client_config.set_client_type(SYNCHRONOUS_CLIENT);
client_config.set_enable_ssl(false);
@ -63,8 +68,11 @@ static void RunSynchronousStreamingPingPong() {
const auto result =
RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
ReportQPS(result);
ReportLatency(result);
std::set<ReportType> types;
types.insert(grpc::testing::ReportType::REPORT_QPS);
types.insert(grpc::testing::ReportType::REPORT_LATENCY);
reporters_registry.Report({client_config, server_config, result}, types);
}
} // namespace testing

@ -31,6 +31,8 @@
*
*/
#include <set>
#include <grpc/support/log.h>
#include <signal.h>
@ -47,6 +49,9 @@ static const int BENCHMARK = 10;
static void RunSynchronousUnaryPingPong() {
gpr_log(GPR_INFO, "Running Synchronous Unary Ping Pong");
ReportersRegistry reporters_registry;
reporters_registry.Register(new GprLogReporter("LogReporter"));
ClientConfig client_config;
client_config.set_client_type(SYNCHRONOUS_CLIENT);
client_config.set_enable_ssl(false);
@ -63,8 +68,10 @@ static void RunSynchronousUnaryPingPong() {
const auto result =
RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
ReportQPS(result);
ReportLatency(result);
std::set<ReportType> types;
types.insert(grpc::testing::ReportType::REPORT_QPS);
types.insert(grpc::testing::ReportType::REPORT_LATENCY);
reporters_registry.Report({client_config, server_config, result}, types);
}
} // namespace testing

Loading…
Cancel
Save