Merge branch 'master' of https://github.com/grpc/grpc into new_tsi

reviewable/pr10522/r3
jiangtaoli2016 8 years ago
commit f1a23b0f60
  1. 7
      CMakeLists.txt
  2. 4
      Makefile
  3. 1
      memory_usage.csv
  4. 6
      src/proto/grpc/testing/BUILD
  5. 6
      src/proto/grpc/testing/services.proto
  6. 7
      templates/CMakeLists.txt.template
  7. 46
      test/cpp/microbenchmarks/bm_cq_multiple_threads.cc
  8. 15
      test/cpp/qps/benchmark_config.cc
  9. 35
      test/cpp/qps/report.cc
  10. 18
      test/cpp/qps/report.h

@ -5,9 +5,6 @@
# This file can be regenerated from the template by running
# tools/buildgen/generate_projects.sh
#
# Additionally, this is currently very experimental, and unsupported.
# Further work will happen on that file.
#
# Copyright 2015, Google Inc.
# All rights reserved.
#
@ -308,7 +305,7 @@ function(protobuf_generate_grpc_cpp)
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)
file(RELATIVE_PATH REL_FIL ${CMAKE_SOURCE_DIR} ${ABS_FIL})
file(RELATIVE_PATH REL_FIL ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL})
get_filename_component(REL_DIR ${REL_FIL} DIRECTORY)
set(RELFIL_WE "${REL_DIR}/${FIL_WE}")
@ -324,7 +321,7 @@ function(protobuf_generate_grpc_cpp)
${_protobuf_include_path}
${REL_FIL}
DEPENDS ${ABS_FIL} ${_gRPC_PROTOBUF_PROTOC} grpc_cpp_plugin
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "Running gRPC C++ protocol buffer compiler on ${FIL}"
VERBATIM)

@ -2388,12 +2388,12 @@ ifeq ($(NO_PROTOC),true)
$(GENDIR)/src/proto/grpc/testing/services.pb.cc: protoc_dep_error
$(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc: protoc_dep_error
else
$(GENDIR)/src/proto/grpc/testing/services.pb.cc: src/proto/grpc/testing/services.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc
$(GENDIR)/src/proto/grpc/testing/services.pb.cc: src/proto/grpc/testing/services.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc
$(E) "[PROTOC] Generating protobuf CC file from $<"
$(Q) mkdir -p `dirname $@`
$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --cpp_out=$(GENDIR) $<
$(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc: src/proto/grpc/testing/services.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc
$(GENDIR)/src/proto/grpc/testing/services.grpc.pb.cc: src/proto/grpc/testing/services.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/control.pb.cc $(GENDIR)/src/proto/grpc/testing/control.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.pb.cc $(GENDIR)/src/proto/grpc/testing/stats.grpc.pb.cc
$(E) "[GRPC] Generating gRPC's protobuf service CC file from $<"
$(Q) mkdir -p `dirname $@`
$(Q) $(PROTOC) -Ithird_party/protobuf/src -I. --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(PROTOC_PLUGINS_DIR)/grpc_cpp_plugin$(EXECUTABLE_SUFFIX) $<

@ -1 +0,0 @@
5678.136000,512736,4195801,5849.316778,204605,,
unable to load file from head commit

@ -83,7 +83,11 @@ grpc_proto_library(
grpc_proto_library(
name = "services_proto",
srcs = ["services.proto"],
deps = ["control_proto", "messages_proto"],
deps = [
"control_proto",
"messages_proto",
"stats_proto",
],
)
grpc_proto_library(

@ -33,6 +33,7 @@ syntax = "proto3";
import "src/proto/grpc/testing/messages.proto";
import "src/proto/grpc/testing/control.proto";
import "src/proto/grpc/testing/stats.proto";
package grpc.testing;
@ -69,3 +70,8 @@ service WorkerService {
// Quit this worker
rpc QuitWorker(Void) returns (Void);
}
service ReportQpsScenarioService {
// Report results of a QPS test benchmark scenario.
rpc ReportScenario(ScenarioResult) returns (Void);
}

@ -7,9 +7,6 @@
# This file can be regenerated from the template by running
# tools/buildgen/generate_projects.sh
#
# Additionally, this is currently very experimental, and unsupported.
# Further work will happen on that file.
#
# Copyright 2015, Google Inc.
# All rights reserved.
#
@ -353,7 +350,7 @@
foreach(FIL <%text>${ARGN}</%text>)
get_filename_component(ABS_FIL <%text>${FIL}</%text> ABSOLUTE)
get_filename_component(FIL_WE <%text>${FIL}</%text> NAME_WE)
file(RELATIVE_PATH REL_FIL <%text>${CMAKE_SOURCE_DIR}</%text> <%text>${ABS_FIL}</%text>)
file(RELATIVE_PATH REL_FIL <%text>${CMAKE_CURRENT_SOURCE_DIR}</%text> <%text>${ABS_FIL}</%text>)
get_filename_component(REL_DIR <%text>${REL_FIL}</%text> DIRECTORY)
set(RELFIL_WE "<%text>${REL_DIR}/${FIL_WE}</%text>")
@ -369,7 +366,7 @@
<%text>${_protobuf_include_path}</%text>
<%text>${REL_FIL}</%text>
DEPENDS <%text>${ABS_FIL}</%text> <%text>${_gRPC_PROTOBUF_PROTOC}</%text> grpc_cpp_plugin
WORKING_DIRECTORY <%text>${CMAKE_SOURCE_DIR}</%text>
WORKING_DIRECTORY <%text>${CMAKE_CURRENT_SOURCE_DIR}</%text>
COMMENT "Running gRPC C++ protocol buffer compiler on <%text>${FIL}</%text>"
VERBATIM)

@ -36,6 +36,8 @@
#include <atomic>
#include <grpc/grpc.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include "test/cpp/microbenchmarks/helpers.h"
extern "C" {
@ -51,13 +53,10 @@ struct grpc_pollset {
namespace grpc {
namespace testing {
static void* make_tag(int i) { return (void*)(intptr_t)i; }
static void* g_tag = (void*)(intptr_t)10; // Some random number
static grpc_completion_queue* g_cq;
static grpc_event_engine_vtable g_vtable;
static __thread int g_thread_idx;
static __thread grpc_cq_completion g_cq_completion;
static void pollset_shutdown(grpc_exec_ctx* exec_ctx, grpc_pollset* ps,
grpc_closure* closure) {
grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE);
@ -76,15 +75,18 @@ static grpc_error* pollset_kick(grpc_pollset* p, grpc_pollset_worker* worker) {
/* Callback when the tag is dequeued from the completion queue. Does nothing */
static void cq_done_cb(grpc_exec_ctx* exec_ctx, void* done_arg,
grpc_cq_completion* cq_completion) {}
grpc_cq_completion* cq_completion) {
gpr_free(cq_completion);
}
/* Queues a completion tag. ZERO polling overhead */
static grpc_error* pollset_work(grpc_exec_ctx* exec_ctx, grpc_pollset* ps,
grpc_pollset_worker** worker, gpr_timespec now,
gpr_timespec deadline) {
gpr_mu_unlock(&ps->mu);
grpc_cq_end_op(exec_ctx, g_cq, make_tag(g_thread_idx), GRPC_ERROR_NONE,
cq_done_cb, NULL, &g_cq_completion);
grpc_cq_begin_op(g_cq, g_tag);
grpc_cq_end_op(exec_ctx, g_cq, g_tag, GRPC_ERROR_NONE, cq_done_cb, NULL,
(grpc_cq_completion*)gpr_malloc(sizeof(grpc_cq_completion)));
grpc_exec_ctx_flush(exec_ctx);
gpr_mu_lock(&ps->mu);
return GRPC_ERROR_NONE;
@ -109,26 +111,42 @@ static void setup() {
g_cq = grpc_completion_queue_create(NULL);
}
static void teardown() {
grpc_completion_queue_shutdown(g_cq);
grpc_completion_queue_destroy(g_cq);
}
/* A few notes about Multi-threaded benchmarks:
Setup:
The benchmark framework ensures that none of the threads proceed beyond the
state.KeepRunning() call unless all the threads have called state.keepRunning
atleast once. So it is safe to do the initialization in one of the threads
before state.KeepRunning() is called.
Teardown:
The benchmark framework also ensures that no thread is running the benchmark
code (i.e the code between two successive calls of state.KeepRunning()) if
state.KeepRunning() returns false. So it is safe to do the teardown in one
of the threads after state.keepRunning() returns false.
*/
static void BM_Cq_Throughput(benchmark::State& state) {
TrackCounters track_counters;
gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
if (state.thread_index == 0) {
setup();
}
while (state.KeepRunning()) {
g_thread_idx = state.thread_index;
void* dummy_tag = make_tag(g_thread_idx);
grpc_cq_begin_op(g_cq, dummy_tag);
grpc_completion_queue_next(g_cq, deadline, NULL);
GPR_ASSERT(grpc_completion_queue_next(g_cq, deadline, NULL).type ==
GRPC_OP_COMPLETE);
}
state.SetItemsProcessed(state.iterations());
if (state.thread_index == 0) {
grpc_completion_queue_shutdown(g_cq);
grpc_completion_queue_destroy(g_cq);
teardown();
}
track_counters.Finish(state);

@ -33,6 +33,9 @@
#include "test/cpp/qps/benchmark_config.h"
#include <gflags/gflags.h>
#include <grpc++/create_channel.h>
#include <grpc++/security/credentials.h>
#include <grpc/support/log.h>
DEFINE_bool(enable_log_reporter, true,
"Enable reporting of benchmark results through GprLog");
@ -51,6 +54,11 @@ DEFINE_string(server_address, "localhost:50052",
DEFINE_string(tag, "", "Optional tag for the test");
DEFINE_string(rpc_reporter_server_address, "",
"Server address for rpc reporter to send results to");
DEFINE_bool(enable_rpc_reporter, false, "Enable use of RPC reporter");
// In some distros, gflags is in the namespace google, and in some others,
// in gflags. This hack is enabling us to find both.
namespace google {}
@ -75,6 +83,13 @@ static std::shared_ptr<Reporter> InitBenchmarkReporters() {
composite_reporter->add(std::unique_ptr<Reporter>(
new JsonReporter("JsonReporter", FLAGS_scenario_result_file)));
}
if (FLAGS_enable_rpc_reporter) {
GPR_ASSERT(!FLAGS_rpc_reporter_server_address.empty());
composite_reporter->add(std::unique_ptr<Reporter>(new RpcReporter(
"RpcReporter",
grpc::CreateChannel(FLAGS_rpc_reporter_server_address,
grpc::InsecureChannelCredentials()))));
}
return std::shared_ptr<Reporter>(composite_reporter);
}

@ -40,6 +40,9 @@
#include "test/cpp/qps/parse_json.h"
#include "test/cpp/qps/stats.h"
#include <grpc++/client_context.h>
#include "src/proto/grpc/testing/services.grpc.pb.h"
namespace grpc {
namespace testing {
@ -142,5 +145,37 @@ void JsonReporter::ReportCpuUsage(const ScenarioResult& result) {
// NOP - all reporting is handled by ReportQPS.
}
void RpcReporter::ReportQPS(const ScenarioResult& result) {
grpc::ClientContext context;
grpc::Status status;
Void dummy;
gpr_log(GPR_INFO, "RPC reporter sending scenario result to server");
status = stub_->ReportScenario(&context, result, &dummy);
if (status.ok()) {
gpr_log(GPR_INFO, "RpcReporter report RPC success!");
} else {
gpr_log(GPR_ERROR, "RpcReporter report RPC: code: %d. message: %s",
status.error_code(), status.error_message().c_str());
}
}
void RpcReporter::ReportQPSPerCore(const ScenarioResult& result) {
// NOP - all reporting is handled by ReportQPS.
}
void RpcReporter::ReportLatency(const ScenarioResult& result) {
// NOP - all reporting is handled by ReportQPS.
}
void RpcReporter::ReportTimes(const ScenarioResult& result) {
// NOP - all reporting is handled by ReportQPS.
}
void RpcReporter::ReportCpuUsage(const ScenarioResult& result) {
// NOP - all reporting is handled by ReportQPS.
}
} // namespace testing
} // namespace grpc

@ -42,6 +42,9 @@
#include "test/cpp/qps/driver.h"
#include <grpc++/channel.h>
#include "src/proto/grpc/testing/services.grpc.pb.h"
namespace grpc {
namespace testing {
@ -124,6 +127,21 @@ class JsonReporter : public Reporter {
const string report_file_;
};
class RpcReporter : public Reporter {
public:
RpcReporter(const string& name, std::shared_ptr<grpc::Channel> channel)
: Reporter(name), stub_(ReportQpsScenarioService::NewStub(channel)) {}
private:
void ReportQPS(const ScenarioResult& result) override;
void ReportQPSPerCore(const ScenarioResult& result) override;
void ReportLatency(const ScenarioResult& result) override;
void ReportTimes(const ScenarioResult& result) override;
void ReportCpuUsage(const ScenarioResult& result) override;
std::unique_ptr<ReportQpsScenarioService::Stub> stub_;
};
} // namespace testing
} // namespace grpc

Loading…
Cancel
Save