Collecting failure status progress

reviewable/pr8239/r2
Craig Tiller 9 years ago
parent db1a5cce47
commit 0c80c7d12c
  1. 2
      include/grpc++/buffer_pool.h
  2. 3
      src/cpp/common/buffer_pool.cc
  3. 10
      src/proto/grpc/testing/control.proto
  4. 8
      src/proto/grpc/testing/stats.proto
  5. 36
      test/cpp/qps/client.h
  6. 24
      test/cpp/qps/driver.cc
  7. 6
      test/cpp/qps/server_async.cc
  8. 6
      test/cpp/qps/server_sync.cc
  9. 15
      tools/run_tests/performance/scenario_config.py
  10. 168
      tools/run_tests/tests.json

@ -54,7 +54,7 @@ class BufferPool GRPC_FINAL {
/// current size of the pool, memory usage will be monotonically decreased
/// until it falls under new_size. No time bound is given for this to occur
/// however.
void Resize(size_t new_size);
BufferPool& Resize(size_t new_size);
grpc_buffer_pool* c_buffer_pool() const { return impl_; }

@ -43,8 +43,9 @@ BufferPool::BufferPool(const grpc::string& name)
BufferPool::~BufferPool() { grpc_buffer_pool_unref(impl_); }
void BufferPool::Resize(size_t new_size) {
BufferPool& BufferPool::Resize(size_t new_size) {
grpc_buffer_pool_resize(impl_, new_size);
return *this;
}
} // namespace grpc

@ -137,6 +137,11 @@ message ServerConfig {
// If we use an OTHER_SERVER client_type, this string gives more detail
string other_server_api = 11;
// c++-only options (for now) --------------------------------
// Buffer pool size (no buffer pool specified if unset)
int32 buffer_pool_size = 1001;
}
message ServerArgs {
@ -213,6 +218,9 @@ message ScenarioResultSummary
double latency_95 = 9;
double latency_99 = 10;
double latency_999 = 11;
int64 successful_requests = 12;
int64 failed_requests = 13;
}
// Results of a single benchmark scenario.
@ -232,4 +240,6 @@ message ScenarioResult {
// Information on success or failure of each worker
repeated bool client_success = 7;
repeated bool server_success = 8;
// Number of failed requests (one row per status code seen)
repeated RequestResultCount request_results = 9;
}

@ -59,6 +59,11 @@ message HistogramData {
double count = 6;
}
message RequestResultCount {
int32 status_code = 1;
int64 count = 2;
}
message ClientStats {
// Latency histogram. Data points are in nanoseconds.
HistogramData latencies = 1;
@ -67,4 +72,7 @@ message ClientStats {
double time_elapsed = 2;
double time_user = 3;
double time_system = 4;
// Number of failed requests (one row per status code seen)
repeated RequestResultCount request_results = 5;
}

@ -36,6 +36,7 @@
#include <condition_variable>
#include <mutex>
#include <unordered_map>
#include <vector>
#include <grpc++/channel.h>
@ -127,6 +128,16 @@ class HistogramEntry GRPC_FINAL {
double value_;
};
typedef std::unordered_map<int, int64_t> StatusHistogram;
inline void MergeStatusHistogram(const StatusHistogram& from,
StatusHistogram* to) {
for (StatusHistogram::const_iterator it = from.begin(); it != from.end();
++it) {
(*to)[it->first] += it->second;
}
}
class Client {
public:
Client()
@ -139,6 +150,7 @@ class Client {
ClientStats Mark(bool reset) {
Histogram latencies;
StatusHistogram statuses;
UsageTimer::Result timer_result;
MaybeStartRequests();
@ -146,27 +158,36 @@ class Client {
// avoid std::vector for old compilers that expect a copy constructor
if (reset) {
Histogram* to_merge = new Histogram[threads_.size()];
StatusHistogram* to_merge_status = new StatusHistogram[threads_.size()];
for (size_t i = 0; i < threads_.size(); i++) {
threads_[i]->BeginSwap(&to_merge[i]);
threads_[i]->BeginSwap(&to_merge[i], &to_merge_status[i]);
}
std::unique_ptr<UsageTimer> timer(new UsageTimer);
timer_.swap(timer);
for (size_t i = 0; i < threads_.size(); i++) {
threads_[i]->EndSwap();
latencies.Merge(to_merge[i]);
MergeStatusHistogram(to_merge_status[i], &statuses);
}
delete[] to_merge;
delete[] to_merge_status;
timer_result = timer->Mark();
} else {
// merge snapshots of each thread histogram
for (size_t i = 0; i < threads_.size(); i++) {
threads_[i]->MergeStatsInto(&latencies);
threads_[i]->MergeStatsInto(&latencies, &statuses);
}
timer_result = timer_->Mark();
}
ClientStats stats;
latencies.FillProto(stats.mutable_latencies());
for (StatusHistogram::const_iterator it = statuses.begin();
it != statuses.end(); ++it) {
RequestResultCount* rrc = stats.add_request_results();
rrc->set_status_code(it->first);
rrc->set_count(it->second);
}
stats.set_time_elapsed(timer_result.wall);
stats.set_time_system(timer_result.system);
stats.set_time_user(timer_result.user);
@ -258,16 +279,16 @@ class Client {
~Thread() { impl_.join(); }
void BeginSwap(Histogram* n) {
void BeginSwap(Histogram* n, StatusHistogram* s) {
std::lock_guard<std::mutex> g(mu_);
n->Swap(&histogram_);
s->swap(statuses_);
}
void EndSwap() {}
void MergeStatsInto(Histogram* hist) {
void MergeStatsInto(Histogram* hist, StatusHistogram* s) {
std::unique_lock<std::mutex> g(mu_);
hist->Merge(histogram_);
MergeStatusHistogram(statuses_, s);
}
private:
@ -304,6 +325,7 @@ class Client {
std::mutex mu_;
Histogram histogram_;
StatusHistogram statuses_;
Client* client_;
const size_t idx_;
std::thread impl_;

@ -157,6 +157,19 @@ static void postprocess_scenario_result(ScenarioResult* result) {
result->mutable_summary()->set_server_user_time(server_user_time);
result->mutable_summary()->set_client_system_time(client_system_time);
result->mutable_summary()->set_client_user_time(client_user_time);
if (result->request_results_size() > 0) {
int64_t successes = 0;
int64_t failures = 0;
for (int i = 0; i < result->request_results_size(); i++) {
RequestResultCount rrc = result->request_results(i);
if (rrc.status_code() == 0) {
successes += rrc.count();
} else {
failures += rrc.count();
}
}
}
}
// Namespace for classes and functions used only in RunScenario
@ -444,6 +457,7 @@ std::unique_ptr<ScenarioResult> RunScenario(
// Finish a run
std::unique_ptr<ScenarioResult> result(new ScenarioResult);
Histogram merged_latencies;
std::unordered_map<int, int64_t> merged_statuses;
gpr_log(GPR_INFO, "Finishing clients");
for (size_t i = 0; i < num_clients; i++) {
@ -462,6 +476,10 @@ std::unique_ptr<ScenarioResult> RunScenario(
gpr_log(GPR_INFO, "Received final status from client %zu", i);
const auto& stats = client_status.stats();
merged_latencies.MergeProto(stats.latencies());
for (int i = 0; i < stats.request_results_size(); i++) {
merged_statuses[stats.request_results(i).status_code()] +=
stats.request_results(i).count();
}
result->add_client_stats()->CopyFrom(stats);
// That final status should be the last message on the client stream
GPR_ASSERT(!client->stream->Read(&client_status));
@ -481,6 +499,12 @@ std::unique_ptr<ScenarioResult> RunScenario(
delete[] clients;
merged_latencies.FillProto(result->mutable_latencies());
for (std::unordered_map<int, int64_t>::iterator it = merged_statuses.begin();
it != merged_statuses.end(); ++it) {
RequestResultCount* rrc = result->add_request_results();
rrc->set_status_code(it->first);
rrc->set_count(it->second);
}
gpr_log(GPR_INFO, "Finishing servers");
for (size_t i = 0; i < num_servers; i++) {

@ -37,6 +37,7 @@
#include <mutex>
#include <thread>
#include <grpc++/buffer_pool.h>
#include <grpc++/generic/async_generic_service.h>
#include <grpc++/security/server_credentials.h>
#include <grpc++/server.h>
@ -95,6 +96,11 @@ class AsyncQpsServerTest : public Server {
srv_cqs_.emplace_back(builder.AddCompletionQueue());
}
if (config.buffer_pool_size() > 0) {
builder.SetBufferPool(
BufferPool("AsyncQpsServerTest").Resize(config.buffer_pool_size()));
}
server_ = builder.BuildAndStart();
using namespace std::placeholders;

@ -33,6 +33,7 @@
#include <thread>
#include <grpc++/buffer_pool.h>
#include <grpc++/security/server_credentials.h>
#include <grpc++/server.h>
#include <grpc++/server_builder.h>
@ -93,6 +94,11 @@ class SynchronousServer GRPC_FINAL : public grpc::testing::Server {
Server::CreateServerCredentials(config));
gpr_free(server_address);
if (config.buffer_pool_size() > 0) {
builder.SetBufferPool(
BufferPool("AsyncQpsServerTest").Resize(config.buffer_pool_size()));
}
builder.RegisterService(&service_);
impl_ = builder.BuildAndStart();

@ -114,7 +114,8 @@ def _ping_pong_scenario(name, rpc_type,
warmup_seconds=WARMUP_SECONDS,
categories=DEFAULT_CATEGORIES,
channels=None,
outstanding=None):
outstanding=None,
buffer_pool_size=None):
"""Creates a basic ping pong scenario."""
scenario = {
'name': name,
@ -141,6 +142,8 @@ def _ping_pong_scenario(name, rpc_type,
'warmup_seconds': warmup_seconds,
'benchmark_seconds': BENCHMARK_SECONDS
}
if buffer_pool_size:
scenario['server_config']['buffer_pool_size'] = buffer_pool_size
if use_generic_payload:
if server_type != 'ASYNC_GENERIC_SERVER':
raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.')
@ -238,6 +241,16 @@ class CXXLanguage:
secure=secure,
categories=smoketest_categories+[SCALABLE])
yield _ping_pong_scenario(
'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_buffer_pool' % (synchronicity, rpc_type, secstr),
rpc_type=rpc_type.upper(),
client_type='%s_CLIENT' % synchronicity.upper(),
server_type='%s_SERVER' % synchronicity.upper(),
unconstrained_client=synchronicity,
secure=secure,
categories=smoketest_categories+[SCALABLE],
buffer_pool_size=500*1024)
for channels in geometric_progression(1, 20000, math.sqrt(10)):
for outstanding in geometric_progression(1, 200000, math.sqrt(10)):
if synchronicity == 'sync' and outstanding > 1200: continue

@ -32065,6 +32065,27 @@
"shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_buffer_pool\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"buffer_pool_size\": 512000, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
],
"boringssl": true,
"ci_platforms": [
"linux"
],
"cpu_cost": 8,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux"
],
"shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_secure_500kib_buffer_pool",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
@ -32107,6 +32128,27 @@
"shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_buffer_pool\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"buffer_pool_size\": 512000, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
],
"boringssl": true,
"ci_platforms": [
"linux"
],
"cpu_cost": 8,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux"
],
"shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure_500kib_buffer_pool",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
@ -32149,6 +32191,27 @@
"shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_buffer_pool\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"buffer_pool_size\": 512000, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
],
"boringssl": true,
"ci_platforms": [
"linux"
],
"cpu_cost": 8,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux"
],
"shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_secure_500kib_buffer_pool",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
@ -32191,6 +32254,27 @@
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_buffer_pool\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"buffer_pool_size\": 512000, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
],
"boringssl": true,
"ci_platforms": [
"linux"
],
"cpu_cost": 8,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux"
],
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure_500kib_buffer_pool",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
@ -32275,6 +32359,27 @@
"shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_buffer_pool\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"buffer_pool_size\": 512000, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
],
"boringssl": true,
"ci_platforms": [
"linux"
],
"cpu_cost": 8,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux"
],
"shortname": "json_run_localhost:cpp_protobuf_sync_unary_qps_unconstrained_insecure_500kib_buffer_pool",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
@ -32317,6 +32422,27 @@
"shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_buffer_pool\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"buffer_pool_size\": 512000, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
],
"boringssl": true,
"ci_platforms": [
"linux"
],
"cpu_cost": 8,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux"
],
"shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure_500kib_buffer_pool",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
@ -32359,6 +32485,27 @@
"shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_buffer_pool\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"buffer_pool_size\": 512000, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 16, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
],
"boringssl": true,
"ci_platforms": [
"linux"
],
"cpu_cost": 8,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux"
],
"shortname": "json_run_localhost:cpp_protobuf_sync_streaming_qps_unconstrained_insecure_500kib_buffer_pool",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
@ -32401,6 +32548,27 @@
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure",
"timeout_seconds": 180
},
{
"args": [
"--scenarios_json",
"{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_buffer_pool\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"buffer_pool_size\": 512000, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}"
],
"boringssl": true,
"ci_platforms": [
"linux"
],
"cpu_cost": 8,
"defaults": "boringssl",
"exclude_configs": [],
"flaky": false,
"language": "c++",
"name": "json_run_localhost",
"platforms": [
"linux"
],
"shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure_500kib_buffer_pool",
"timeout_seconds": 180
},
{
"args": [
"test/core/end2end/fuzzers/api_fuzzer_corpus/00.bin"

Loading…
Cancel
Save