From 922ea81877fc8b0676ae90d59a3caad9954e2feb Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Thu, 4 Jun 2015 17:32:31 -0700 Subject: [PATCH 01/63] QPS, latencies recorded with authentication --- Makefile | 39 +++++++--- build.json | 6 +- test/cpp/qps/report.cc | 103 +++++++++++++++++++++++++ test/cpp/qps/report.h | 14 ++++ test/cpp/qps/run_authenticated_test.sh | 87 +++++++++++++++++++++ test/cpp/qps/user_data.proto | 99 ++++++++++++++++++++++++ test/cpp/qps/user_data_client.cc | 100 ++++++++++++++++++++++++ test/cpp/qps/user_data_client.h | 88 +++++++++++++++++++++ test/cpp/util/benchmark_config.cc | 6 ++ 9 files changed, 532 insertions(+), 10 deletions(-) create mode 100755 test/cpp/qps/run_authenticated_test.sh create mode 100644 test/cpp/qps/user_data.proto create mode 100644 test/cpp/qps/user_data_client.cc create mode 100644 test/cpp/qps/user_data_client.h diff --git a/Makefile b/Makefile index 13170c7227a..fbe2e4c3c2c 100644 --- a/Makefile +++ b/Makefile @@ -2457,6 +2457,21 @@ $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc: test/cpp/qps/qpstest.proto $(PROTOBUF $(Q) $(PROTOC) --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $< endif +ifeq ($(NO_PROTOC),true) +$(GENDIR)/test/cpp/qps/user_data.pb.cc: protoc_dep_error +$(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc: protoc_dep_error +else +$(GENDIR)/test/cpp/qps/user_data.pb.cc: test/cpp/qps/user_data.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) + $(E) "[PROTOC] Generating protobuf CC file from $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(PROTOC) --cpp_out=$(GENDIR) $< + +$(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc: test/cpp/qps/user_data.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) + $(E) "[GRPC] Generating gRPC's protobuf service CC file from $<" + $(Q) mkdir -p `dirname $@` + $(Q) $(PROTOC) --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $< +endif + ifeq ($(NO_PROTOC),true) $(GENDIR)/test/cpp/util/echo.pb.cc: protoc_dep_error $(GENDIR)/test/cpp/util/echo.grpc.pb.cc: protoc_dep_error @@ -3452,7 +3467,9 @@ endif LIBGRPC++_BENCHMARK_CONFIG_SRC = \ $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc \ + $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc \ test/cpp/qps/report.cc \ + test/cpp/qps/user_data_client.cc \ test/cpp/util/benchmark_config.cc \ @@ -3497,8 +3514,9 @@ ifneq ($(NO_DEPS),true) -include $(LIBGRPC++_BENCHMARK_CONFIG_OBJS:.o=.dep) endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/user_data_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc LIBGRPC++_TEST_CONFIG_SRC = \ @@ -4011,6 +4029,7 @@ $(OBJDIR)/$(CONFIG)/examples/pubsub/subscriber.o: $(GENDIR)/examples/pubsub/labe LIBQPS_SRC = \ $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc \ + $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc \ test/cpp/qps/client_async.cc \ test/cpp/qps/client_sync.cc \ test/cpp/qps/driver.cc \ @@ -4018,6 +4037,7 @@ LIBQPS_SRC = \ test/cpp/qps/server_async.cc \ test/cpp/qps/server_sync.cc \ test/cpp/qps/timer.cc \ + test/cpp/qps/user_data_client.cc \ LIBQPS_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBQPS_SRC)))) @@ -4061,13 +4081,14 @@ ifneq ($(NO_DEPS),true) -include $(LIBQPS_OBJS:.o=.dep) endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/timer.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/timer.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/user_data_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc LIBGRPC_CSHARP_EXT_SRC = \ diff --git a/build.json b/build.json index 084645937b7..0b68f876f9f 100644 --- a/build.json +++ b/build.json @@ -537,7 +537,9 @@ "language": "c++", "src": [ "test/cpp/qps/qpstest.proto", + "test/cpp/qps/user_data.proto", "test/cpp/qps/report.cc", + "test/cpp/qps/user_data_client.cc", "test/cpp/util/benchmark_config.cc" ] }, @@ -707,13 +709,15 @@ ], "src": [ "test/cpp/qps/qpstest.proto", + "test/cpp/qps/user_data.proto", "test/cpp/qps/client_async.cc", "test/cpp/qps/client_sync.cc", "test/cpp/qps/driver.cc", "test/cpp/qps/qps_worker.cc", "test/cpp/qps/server_async.cc", "test/cpp/qps/server_sync.cc", - "test/cpp/qps/timer.cc" + "test/cpp/qps/timer.cc", + "test/cpp/qps/user_data_client.cc" ], "deps": [ "grpc_test_util", diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index e116175e3b3..b4229889add 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -35,6 +35,7 @@ #include #include "test/cpp/qps/stats.h" +#include "user_data_client.h" namespace grpc { namespace testing { @@ -120,5 +121,107 @@ void GprLogReporter::ReportTimes(const ScenarioResult& result) const { [](ResourceUsage u) { return u.wall_time; })); } +UserDataClient userDataClient(grpc::CreateChannel("localhost:50052", grpc::InsecureCredentials(), + ChannelArguments())); + +//Leaderboard Reported implementation. +void UserDatabaseReporter::ReportQPS(const ScenarioResult& result) const { + double qps = result.latencies.Count() / + average(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); + + userDataClient.setAccessToken(access_token_); + userDataClient.setQPS(qps); + + int userDataState = userDataClient.sendDataIfReady(); + + switch(userDataState) { + case 1: + gpr_log(GPR_INFO, "Data sent to user database successfully"); + break; + case -1: + gpr_log(GPR_INFO, "Data could not be sent to user database"); + break; + } +} + +void UserDatabaseReporter::ReportQPSPerCore(const ScenarioResult& result, + const ServerConfig& server_config) const { + double qps = result.latencies.Count() / + average(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); + + double qpsPerCore = qps / server_config.threads(); + + userDataClient.setAccessToken(access_token_); + //TBD + userDataClient.setQPSPerCore(qpsPerCore); + + int userDataState = userDataClient.sendDataIfReady(); + + switch(userDataState) { + case 1: + gpr_log(GPR_INFO, "Data sent to user database successfully"); + break; + case -1: + gpr_log(GPR_INFO, "Data could not be sent to user database"); + break; + } +} + +void UserDatabaseReporter::ReportLatency(const ScenarioResult& result) const { + userDataClient.setAccessToken(access_token_); + userDataClient.setLatencies(result.latencies.Percentile(50) / 1000, + result.latencies.Percentile(90) / 1000, + result.latencies.Percentile(95) / 1000, + result.latencies.Percentile(99) / 1000, + result.latencies.Percentile(99.9) / 1000); + + int userDataState = userDataClient.sendDataIfReady(); + + switch(userDataState) { + case 1: + gpr_log(GPR_INFO, "Data sent to user database successfully"); + break; + case -1: + gpr_log(GPR_INFO, "Data could not be sent to user database"); + break; + } +} + +void UserDatabaseReporter::ReportTimes(const ScenarioResult& result) const { + double serverSystemTime = 100.0 * sum(result.server_resources, + [](ResourceUsage u) { return u.system_time; }) / + sum(result.server_resources, + [](ResourceUsage u) { return u.wall_time; }); + double serverUserTime = 100.0 * sum(result.server_resources, + [](ResourceUsage u) { return u.user_time; }) / + sum(result.server_resources, + [](ResourceUsage u) { return u.wall_time; }); + double clientSystemTime = 100.0 * sum(result.client_resources, + [](ResourceUsage u) { return u.system_time; }) / + sum(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); + double clientUserTime = 100.0 * sum(result.client_resources, + [](ResourceUsage u) { return u.user_time; }) / + sum(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); + + userDataClient.setAccessToken(access_token_); + userDataClient.setTimes(serverSystemTime, serverUserTime, + clientSystemTime, clientUserTime); + + int userDataState = userDataClient.sendDataIfReady(); + + switch(userDataState) { + case 1: + gpr_log(GPR_INFO, "Data sent to user database successfully"); + break; + case -1: + gpr_log(GPR_INFO, "Data could not be sent to user database"); + break; + } +} + } // namespace testing } // namespace grpc diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 630275ecda2..2e5b627e581 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -106,6 +106,20 @@ class GprLogReporter : public Reporter { void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; }; +/** Reporter for client leaderboard. */ +class UserDatabaseReporter : public Reporter { + public: + UserDatabaseReporter(const string& name, const string& access_token) : Reporter(name), access_token_(access_token) {} + + private: + std::string access_token_; + void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE; + void ReportQPSPerCore(const ScenarioResult& result, + const ServerConfig& config) const GRPC_OVERRIDE; + void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE; + void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; +}; + } // namespace testing } // namespace grpc diff --git a/test/cpp/qps/run_authenticated_test.sh b/test/cpp/qps/run_authenticated_test.sh new file mode 100755 index 00000000000..3cde6494142 --- /dev/null +++ b/test/cpp/qps/run_authenticated_test.sh @@ -0,0 +1,87 @@ +#!/bin/sh + +CLIENT_ID='1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' +CLIENT_SECRET='_HGHXg4DAA59r4w4x8p6ARzD' +GRANT_TYPE='http://oauth.net/grant_type/device/1.0' +ACCESS_TOKENS_DIR='/tmp/auth_lead_access_tokens' +AUTH_TOKEN_LINK='https://www.googleapis.com/oauth2/v3/token' +GOOGLE_ACCOUNTS_LINK='https://accounts.google.com/o/oauth2/device/code' +USER_INFO_LINK='https://www.googleapis.com/oauth2/v1/userinfo' +#Performs first time authentication +#Or re-authentication if refresh token expires +RE_AUTHENTICATE() { + INIT_AUTH_JSON=$(curl -s -d "client_id=$CLIENT_ID&scope=email profile" $GOOGLE_ACCOUNTS_LINK) + + USER_CODE=$(echo $INIT_AUTH_JSON | jq .user_code | sed -e 's/^"//' -e 's/"$//') + echo 'Please use the following user code in the browser:' $USER_CODE + echo + + VERIFICATION_URL=$(echo $INIT_AUTH_JSON | jq '.verification_url' | sed -e 's/^"//' -e 's/"$//') + echo 'Verification URL:' $VERIFICATION_URL + echo + + xdg-open $VERIFICATION_URL + + DEVICE_CODE=$(echo $INIT_AUTH_JSON | jq '.device_code' | sed -e 's/^"//' -e 's/"$//') + INTERVAL=$(echo $INIT_AUTH_JSON | jq '.interval' | sed -e 's/^"//' -e 's/"$//') + + AUTH_JSON=$(curl -s -d "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&code=$DEVICE_CODE&grant_type=$GRANT_TYPE" $AUTH_TOKEN_LINK) + ACCESS_TOKEN=$(echo $AUTH_JSON | jq '.access_token' | sed -e 's/^"//' -e 's/"$//') + + while [ $ACCESS_TOKEN == 'null' ] + do + sleep $INTERVAL + AUTH_JSON=$(curl -s -d "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&code=$DEVICE_CODE&grant_type=$GRANT_TYPE" $AUTH_TOKEN_LINK) + ACCESS_TOKEN=$(echo $AUTH_JSON | jq '.access_token' | sed -e 's/^"//' -e 's/"$//') + done + + USER_DETAILS=$(curl -s $USER_INFO_LINK?access_token=$ACCESS_TOKEN) + USER_ID=$(echo $USER_DETAILS | jq '.email' | sed -e 's/^"//' -e 's/"$//' | awk -F"@" '{print $1}' | sed -e 's/\.//g' | awk '{print tolower($0)}') + echo $AUTH_JSON > $ACCESS_TOKENS_DIR/$USER_ID +} + +#Use existing access token +USE_ACCESS_TOKEN() { + ACCESS_TOKEN=$(jq '.access_token' $ACCESS_TOKENS_DIR/$USER_ID | sed -e 's/^"//' -e 's/"$//') + + USER_DETAILS=$(curl -s $USER_INFO_LINK?access_token=$ACCESS_TOKEN) + + ID=$(echo $USER_DETAILS | jq '.id' | sed -e 's/^"//' -e 's/"$//') + + if [ $ID == 'null' ]; then + REFRESH_ACCESS_TOKEN + fi +} + +#Obtain new access token using refresh token +REFRESH_ACCESS_TOKEN() { + REFRESH_TOKEN=$(jq '.refresh_token' $ACCESS_TOKENS_DIR/$USER_ID | sed -e 's/^"//' -e 's/"$//') + if [ $REFRESH_TOKEN == 'null' ]; then + RE_AUTHENTICATE + else + REFRESH_JSON=$(curl -s -d "refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&grant_type=refresh_token" $AUTH_TOKEN_LINK) + + ACCESS_TOKEN=$(echo $REFRESH_JSON | jq '.access_token') + if [ $ACCESS_TOKEN == 'null' ]; then + RE_AUTHENTICATE + else + NEW_AUTH_JSON=$(jq ".access_token=$ACCESS_TOKEN" $ACCESS_TOKENS_DIR/$USER_ID) + echo $NEW_AUTH_JSON > $ACCESS_TOKENS_DIR/$USER_ID + fi + fi +} + +#create directory to store tokens, if not already present +[ ! -d $ACCESS_TOKENS_DIR ] && mkdir $ACCESS_TOKENS_DIR + +#Convert user entered email id to unique string by converting to splitting on '@' symbol, if present, +#removing '.'s and converting to lowercase +USER_ID=$(echo $2 | awk -F"@" '{print $1}' | sed -e 's/\.//g' | awk '{print tolower($0)}') + +if [ -s $ACCESS_TOKENS_DIR/$USER_ID ]; then + USE_ACCESS_TOKEN +else + RE_AUTHENTICATE +fi + +./$1 --access_token=$ACCESS_TOKEN \ No newline at end of file diff --git a/test/cpp/qps/user_data.proto b/test/cpp/qps/user_data.proto new file mode 100644 index 00000000000..9c941cae2c5 --- /dev/null +++ b/test/cpp/qps/user_data.proto @@ -0,0 +1,99 @@ +// Copyright 2015, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package UserData; + +service UserDataTransfer { + // Sends client info + rpc RecordSingleClientData (SingleUserRecordRequest) returns (SingleUserRecordReply) {} + + rpc RetrieveSingleUserData (SingleUserRetrieveRequest) returns (SingleUserRetrieveReply) {} + + rpc RetrieveAllUsersData (AllUsersRetrieveRequest) returns (AllUsersRetrieveReply) {} +} + +//Metrics to be stored +message Metrics { + required double qps = 1; + required double perc_lat_50 = 2; + required double perc_lat_90 = 3; + required double perc_lat_95 = 4; + required double perc_lat_99 = 5; + required double perc_lat_99_point_9 = 6; +} + +//Timestamped details +message DataDetails { + required string timestamp = 1; + required Metrics metrics = 2; +} + +//User details +message UserDetails { + required string id = 1; + required string name = 2; + required string link = 3; +} + +//Stored to database +message SingleUserDetails { + repeated DataDetails data_details = 1; + required UserDetails user_details = 2; +} + +//Request for storing a single user's data +message SingleUserRecordRequest { + required string access_token = 1; + required Metrics metrics = 2; +} + +//Reply to request for storing single user's data +message SingleUserRecordReply { +} + +//Request for retrieving single user's data +message SingleUserRetrieveRequest { + required string client_id = 1; +} + +//Reply for request to retrieve single user's data +message SingleUserRetrieveReply { + required SingleUserDetails details = 1; +} + +//Request for retrieving all users' data +message AllUsersRetrieveReply { + repeated SingleUserDetails user_data = 1; +} + +//Reply to request for retrieving all users' data +message AllUsersRetrieveRequest { +} \ No newline at end of file diff --git a/test/cpp/qps/user_data_client.cc b/test/cpp/qps/user_data_client.cc new file mode 100644 index 00000000000..20ce6d9c09e --- /dev/null +++ b/test/cpp/qps/user_data_client.cc @@ -0,0 +1,100 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "user_data_client.h" + +void UserDataClient::setAccessToken(std::string access_token) { + access_token_ = access_token; +} + +void UserDataClient::setQPS(double QPS) { + QPS_ = QPS; + qpsSet = true; +} + +void UserDataClient::setQPSPerCore(double qpsPerCore) { + //TBD +} + +void UserDataClient::setLatencies(double percentileLatency50, double percentileLatency90, + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) { + percentileLatency50_ = percentileLatency50; + percentileLatency90_ = percentileLatency90; + percentileLatency95_ = percentileLatency95; + percentileLatency99_ = percentileLatency99; + percentileLatency99Point9_ = percentileLatency99Point9; + + latenciesSet = true; +} + +void UserDataClient::setTimes(double serverSystemTime, double serverUserTime, + double clientSystemTime, double clientUserTime) { + //TBD +} + +int UserDataClient::sendDataIfReady() { + if(!(qpsSet && latenciesSet)) + return 0; + + SingleUserRecordRequest singleUserRecordRequest; + singleUserRecordRequest.set_access_token(access_token_); + + Metrics* metrics = singleUserRecordRequest.mutable_metrics(); + metrics->set_qps(QPS_); + metrics->set_perc_lat_50(percentileLatency50_); + metrics->set_perc_lat_90(percentileLatency90_); + metrics->set_perc_lat_95(percentileLatency95_); + metrics->set_perc_lat_99(percentileLatency99_); + metrics->set_perc_lat_99_point_9(percentileLatency99Point9_); + + SingleUserRecordReply singleUserRecordReply; + ClientContext context; + + Status status = stub_->RecordSingleClientData(&context, singleUserRecordRequest, &singleUserRecordReply); + if (status.IsOk()) { + return 1; + } else { + return -1; + } +} + +// Get current date/time, format is YYYY-MM-DD.HH:mm:ss +const std::string currentDateTime() { + time_t now = time(0); + struct tm tstruct; + char buf[80]; + tstruct = *localtime(&now); + + strftime(buf, sizeof(buf), "%Y/%m/%d, %X", &tstruct); + return buf; +} \ No newline at end of file diff --git a/test/cpp/qps/user_data_client.h b/test/cpp/qps/user_data_client.h new file mode 100644 index 00000000000..ec3dcb5d44b --- /dev/null +++ b/test/cpp/qps/user_data_client.h @@ -0,0 +1,88 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include "test/cpp/qps/user_data.grpc.pb.h" + +using grpc::ChannelArguments; +using grpc::ChannelInterface; +using grpc::ClientContext; +using grpc::Status; +using UserData::UserDataTransfer; +using UserData::Metrics; +using UserData::SingleUserRecordRequest; +using UserData::SingleUserRecordReply; + +class UserDataClient { + public: + UserDataClient(std::shared_ptr channel) + : stub_(UserDataTransfer::NewStub(channel)) {} + + ~UserDataClient() {} + + void setAccessToken(std::string access_token); + + void setQPS(double QPS); + + void setQPSPerCore(double qpsPerCore); + + void setLatencies(double percentileLatency50, double percentileLatency90, + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9); + + void setTimes(double serverSystemTime, double serverUserTime, + double clientSystemTime, double clientUserTime); + + int sendDataIfReady(); + + private: + std::unique_ptr stub_; + std::string access_token_; + double QPS_; + double percentileLatency50_; + double percentileLatency90_; + double percentileLatency95_; + double percentileLatency99_; + double percentileLatency99Point9_; + bool qpsSet = false; + bool latenciesSet = false; +}; \ No newline at end of file diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc index 5b3c1daf5db..b68822436db 100644 --- a/test/cpp/util/benchmark_config.cc +++ b/test/cpp/util/benchmark_config.cc @@ -37,6 +37,8 @@ DEFINE_bool(enable_log_reporter, true, "Enable reporting of benchmark results through GprLog"); +DEFINE_string(access_token, "", "Authorizing JSON string for leaderboard"); + // In some distros, gflags is in the namespace google, and in some others, // in gflags. This hack is enabling us to find both. namespace google {} @@ -57,6 +59,10 @@ static std::shared_ptr InitBenchmarkReporters() { composite_reporter->add( std::unique_ptr(new GprLogReporter("LogReporter"))); } + if(!FLAGS_access_token.empty()) + composite_reporter->add( + std::unique_ptr(new UserDatabaseReporter("UserDataReporter", FLAGS_access_token))); + return std::shared_ptr(composite_reporter); } From 39824335ea2d29ab627a8accb8e12c3bd50352ad Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 8 Jun 2015 13:44:51 -0700 Subject: [PATCH 02/63] Newer version --- test/cpp/qps/report.cc | 54 +++--------- test/cpp/qps/report.h | 6 +- test/cpp/qps/run_auth_test.py | 135 ++++++++++++++++++++++++++++++ test/cpp/qps/user_data.proto | 55 ++++++++---- test/cpp/qps/user_data_client.cc | 62 ++++++++------ test/cpp/qps/user_data_client.h | 51 ++++++----- test/cpp/util/benchmark_config.cc | 4 +- 7 files changed, 258 insertions(+), 109 deletions(-) create mode 100644 test/cpp/qps/run_auth_test.py diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index b4229889add..124bdc70fdc 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -130,19 +130,8 @@ void UserDatabaseReporter::ReportQPS(const ScenarioResult& result) const { average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); - userDataClient.setAccessToken(access_token_); userDataClient.setQPS(qps); - - int userDataState = userDataClient.sendDataIfReady(); - - switch(userDataState) { - case 1: - gpr_log(GPR_INFO, "Data sent to user database successfully"); - break; - case -1: - gpr_log(GPR_INFO, "Data could not be sent to user database"); - break; - } + userDataClient.setConfigs(result.client_config, result.server_config); } void UserDatabaseReporter::ReportQPSPerCore(const ScenarioResult& result, @@ -153,40 +142,17 @@ void UserDatabaseReporter::ReportQPSPerCore(const ScenarioResult& result, double qpsPerCore = qps / server_config.threads(); - userDataClient.setAccessToken(access_token_); - //TBD userDataClient.setQPSPerCore(qpsPerCore); - - int userDataState = userDataClient.sendDataIfReady(); - - switch(userDataState) { - case 1: - gpr_log(GPR_INFO, "Data sent to user database successfully"); - break; - case -1: - gpr_log(GPR_INFO, "Data could not be sent to user database"); - break; - } + userDataClient.setConfigs(result.client_config, result.server_config); } void UserDatabaseReporter::ReportLatency(const ScenarioResult& result) const { - userDataClient.setAccessToken(access_token_); userDataClient.setLatencies(result.latencies.Percentile(50) / 1000, - result.latencies.Percentile(90) / 1000, - result.latencies.Percentile(95) / 1000, - result.latencies.Percentile(99) / 1000, - result.latencies.Percentile(99.9) / 1000); - - int userDataState = userDataClient.sendDataIfReady(); - - switch(userDataState) { - case 1: - gpr_log(GPR_INFO, "Data sent to user database successfully"); - break; - case -1: - gpr_log(GPR_INFO, "Data could not be sent to user database"); - break; - } + result.latencies.Percentile(90) / 1000, + result.latencies.Percentile(95) / 1000, + result.latencies.Percentile(99) / 1000, + result.latencies.Percentile(99.9) / 1000); + userDataClient.setConfigs(result.client_config, result.server_config); } void UserDatabaseReporter::ReportTimes(const ScenarioResult& result) const { @@ -207,11 +173,13 @@ void UserDatabaseReporter::ReportTimes(const ScenarioResult& result) const { sum(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); - userDataClient.setAccessToken(access_token_); userDataClient.setTimes(serverSystemTime, serverUserTime, clientSystemTime, clientUserTime); + userDataClient.setConfigs(result.client_config, result.server_config); +} - int userDataState = userDataClient.sendDataIfReady(); +void UserDatabaseReporter::Flush() const { + int userDataState = userDataClient.sendData(access_token_, test_name_); switch(userDataState) { case 1: diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 2e5b627e581..8a1ce2dabc3 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -109,15 +109,19 @@ class GprLogReporter : public Reporter { /** Reporter for client leaderboard. */ class UserDatabaseReporter : public Reporter { public: - UserDatabaseReporter(const string& name, const string& access_token) : Reporter(name), access_token_(access_token) {} + UserDatabaseReporter(const string& name, const string& access_token, + const string& test_name) : Reporter(name), access_token_(access_token), test_name_(test_name) {} + ~UserDatabaseReporter() { Flush(); }; private: std::string access_token_; + std::string test_name_; void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE; void ReportQPSPerCore(const ScenarioResult& result, const ServerConfig& config) const GRPC_OVERRIDE; void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE; void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; + void Flush() const; }; } // namespace testing diff --git a/test/cpp/qps/run_auth_test.py b/test/cpp/qps/run_auth_test.py new file mode 100644 index 00000000000..3ed2a7c9804 --- /dev/null +++ b/test/cpp/qps/run_auth_test.py @@ -0,0 +1,135 @@ +#!/usr/bin/python + +import os +import sys +import re +import urllib2 +import urllib +import json +import time +import subprocess + +CLIENT_ID = '1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' +CLIENT_SECRET = '_HGHXg4DAA59r4w4x8p6ARzD' +GRANT_TYPE = 'http://oauth.net/grant_type/device/1.0' +ACCESS_TOKENS_DIR = '/tmp/auth_lead_access_tokens' +AUTH_TOKEN_LINK = 'https://www.googleapis.com/oauth2/v3/token' +GOOGLE_ACCOUNTS_LINK = 'https://accounts.google.com/o/oauth2/device/code' +USER_INFO_LINK = 'https://www.googleapis.com/oauth2/v1/userinfo' + +def fetchJSON(url, paramDict): + if len(paramDict) == 0: + req = urllib2.Request(url) + else: + data = urllib.urlencode(paramDict) + req = urllib2.Request(url, data) + + try: + response = urllib2.urlopen(req) + result = response.read() + + except urllib2.HTTPError, error: + result = error.read() + + return result + +def getUserInfo(accessToken): + url = USER_INFO_LINK + '?access_token=' + accessToken + paramDict = {} + JSONBody = fetchJSON(url, paramDict) + data = json.loads(JSONBody) + + return data + +def isAccessTokenValid(accessToken): + data = getUserInfo(accessToken); + + if 'id' in data: + return True + else: + return False + +def getUserId(accessToken): + data = getUserInfo(accessToken) + + email = data['email'] + email = email.split('@')[0].lower() + userId = re.sub('[.]', '', email) + + return userId + +def useAccessToken(userTokFile): + with open(userTokFile, "r") as data_file: + data = json.load(data_file) + accessToken = data["access_token"] + + if not isAccessTokenValid(accessToken): + return refreshAccessToken(data["refresh_token"], userTokFile) + + return accessToken + +def refreshAccessToken(refreshToken, userTokFile): + paramDict = {'refresh_token':refreshToken, 'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'grant_type':'refresh_token'} + JSONBody = fetchJSON(AUTH_TOKEN_LINK, paramDict) + data = json.loads(JSONBody) + if not 'access_token' in data: + return reauthenticate() + else: + tokenData = {} + + with open(userTokFile, "r") as data_file: + tokenData = json.load(data_file) + + with open(userTokFile, "w") as data_file: + tokenData['access_token'] = data['access_token'] + json.dump(tokenData, data_file) + + return data['access_token'] + +def reauthenticate(): + paramDict = {'client_id':CLIENT_ID, 'scope':'email profile'} + JSONBody = fetchJSON(GOOGLE_ACCOUNTS_LINK, paramDict) + data = json.loads(JSONBody) + + print 'User authorization required\n' + print 'Please use the following code in you browser: ', data['user_code'] + print 'Verification URL: ', data['verification_url'] + print '\nAwaiting user authorization. May take a few more seconds after authorizing...\n' + + authData = {} + + while not 'access_token' in authData: + authDict = {'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'code':data['device_code'], 'grant_type':GRANT_TYPE} + JSONBody = fetchJSON(AUTH_TOKEN_LINK, authDict) + authData = json.loads(JSONBody) + time.sleep(data['interval']) + + newUserTokFile = ACCESS_TOKENS_DIR + '/' + getUserId(authData['access_token']) + + with open(newUserTokFile, "w") as data_file: + json.dump(authData, data_file) + + return authData['access_token'] + +def main(): + if not os.path.exists(ACCESS_TOKENS_DIR): + os.makedirs(ACCESS_TOKENS_DIR) + + email = sys.argv[2] + email = email.split('@')[0].lower() + userId = re.sub('[.]', '', email) + + userTokFile = ACCESS_TOKENS_DIR + '/' + userId + + accessToken = '' + + if os.path.exists(userTokFile): + accessToken = useAccessToken(userTokFile) + else: + accessToken = reauthenticate() + + testName = sys.argv[1].split('/')[-1] + subprocess.call([sys.argv[1], '--access_token='+accessToken, '--test_name='+testName]) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/test/cpp/qps/user_data.proto b/test/cpp/qps/user_data.proto index 9c941cae2c5..27d0049cf07 100644 --- a/test/cpp/qps/user_data.proto +++ b/test/cpp/qps/user_data.proto @@ -29,7 +29,9 @@ syntax = "proto2"; -package UserData; +import "test/cpp/qps/qpstest.proto"; + +package grpc.testing; service UserDataTransfer { // Sends client info @@ -42,37 +44,56 @@ service UserDataTransfer { //Metrics to be stored message Metrics { - required double qps = 1; - required double perc_lat_50 = 2; - required double perc_lat_90 = 3; - required double perc_lat_95 = 4; - required double perc_lat_99 = 5; - required double perc_lat_99_point_9 = 6; + optional double qps = 1; + optional double qps_per_core = 2; + optional double perc_lat_50 = 3; + optional double perc_lat_90 = 4; + optional double perc_lat_95 = 5; + optional double perc_lat_99 = 6; + optional double perc_lat_99_point_9 = 7; + optional double server_system_time = 8; + optional double server_user_time = 9; + optional double client_system_time = 10; + optional double client_user_time = 11; } //Timestamped details message DataDetails { - required string timestamp = 1; - required Metrics metrics = 2; + optional string timestamp = 1; + optional string test_name = 2; + optional Metrics metrics = 3; + optional ClientConfig client_config = 4; + optional ServerConfig server_config = 5; } //User details message UserDetails { - required string id = 1; - required string name = 2; - required string link = 3; + optional string id = 1; + optional string email = 2; + optional bool verified_email = 3; + optional string name = 4; + optional string given_name = 5; + optional string family_name = 6; + optional string link = 7; + optional string picture = 8; + optional string gender = 9; + optional string locale = 10; + optional string hd = 11; } //Stored to database message SingleUserDetails { repeated DataDetails data_details = 1; - required UserDetails user_details = 2; + optional UserDetails user_details = 2; } //Request for storing a single user's data message SingleUserRecordRequest { - required string access_token = 1; - required Metrics metrics = 2; + optional string access_token = 1; + optional string test_name = 2; + optional Metrics metrics = 3; + optional ClientConfig client_config = 4; + optional ServerConfig server_config = 5; } //Reply to request for storing single user's data @@ -81,12 +102,12 @@ message SingleUserRecordReply { //Request for retrieving single user's data message SingleUserRetrieveRequest { - required string client_id = 1; + optional string user_id = 1; } //Reply for request to retrieve single user's data message SingleUserRetrieveReply { - required SingleUserDetails details = 1; + optional SingleUserDetails details = 1; } //Request for retrieving all users' data diff --git a/test/cpp/qps/user_data_client.cc b/test/cpp/qps/user_data_client.cc index 20ce6d9c09e..50b74276d2e 100644 --- a/test/cpp/qps/user_data_client.cc +++ b/test/cpp/qps/user_data_client.cc @@ -33,17 +33,20 @@ #include "user_data_client.h" -void UserDataClient::setAccessToken(std::string access_token) { - access_token_ = access_token; +namespace grpc { +namespace testing { + +void UserDataClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) { + clientConfig_ = clientConfig; + serverConfig_ = serverConfig; } void UserDataClient::setQPS(double QPS) { QPS_ = QPS; - qpsSet = true; } -void UserDataClient::setQPSPerCore(double qpsPerCore) { - //TBD +void UserDataClient::setQPSPerCore(double QPSPerCore) { + QPSPerCore_ = QPSPerCore; } void UserDataClient::setLatencies(double percentileLatency50, double percentileLatency90, @@ -53,29 +56,37 @@ void UserDataClient::setLatencies(double percentileLatency50, double percentileL percentileLatency95_ = percentileLatency95; percentileLatency99_ = percentileLatency99; percentileLatency99Point9_ = percentileLatency99Point9; - - latenciesSet = true; } void UserDataClient::setTimes(double serverSystemTime, double serverUserTime, double clientSystemTime, double clientUserTime) { - //TBD + serverSystemTime_ = serverSystemTime; + serverUserTime_ = serverUserTime; + clientSystemTime_ = clientSystemTime; + clientUserTime_ = clientUserTime; } -int UserDataClient::sendDataIfReady() { - if(!(qpsSet && latenciesSet)) - return 0; +int UserDataClient::sendData(std::string access_token, std::string test_name) { SingleUserRecordRequest singleUserRecordRequest; - singleUserRecordRequest.set_access_token(access_token_); + singleUserRecordRequest.set_access_token(access_token); + singleUserRecordRequest.set_test_name(test_name); + *(singleUserRecordRequest.mutable_client_config()) = clientConfig_; + *(singleUserRecordRequest.mutable_server_config()) = serverConfig_; Metrics* metrics = singleUserRecordRequest.mutable_metrics(); - metrics->set_qps(QPS_); - metrics->set_perc_lat_50(percentileLatency50_); - metrics->set_perc_lat_90(percentileLatency90_); - metrics->set_perc_lat_95(percentileLatency95_); - metrics->set_perc_lat_99(percentileLatency99_); - metrics->set_perc_lat_99_point_9(percentileLatency99Point9_); + + if(QPS_ != DBL_MIN) metrics->set_qps(QPS_); + if(QPSPerCore_ != DBL_MIN) metrics->set_qps_per_core(QPSPerCore_); + if(percentileLatency50_ != DBL_MIN) metrics->set_perc_lat_50(percentileLatency50_); + if(percentileLatency90_ != DBL_MIN) metrics->set_perc_lat_90(percentileLatency90_); + if(percentileLatency95_ != DBL_MIN) metrics->set_perc_lat_95(percentileLatency95_); + if(percentileLatency99_ != DBL_MIN) metrics->set_perc_lat_99(percentileLatency99_); + if(percentileLatency99Point9_ != DBL_MIN) metrics->set_perc_lat_99_point_9(percentileLatency99Point9_); + if(serverSystemTime_ != DBL_MIN) metrics->set_server_system_time(serverSystemTime_); + if(serverUserTime_ != DBL_MIN) metrics->set_server_user_time(serverUserTime_); + if(clientSystemTime_ != DBL_MIN) metrics->set_client_system_time(clientSystemTime_); + if(clientUserTime_ != DBL_MIN) metrics->set_client_user_time(clientUserTime_); SingleUserRecordReply singleUserRecordReply; ClientContext context; @@ -90,11 +101,14 @@ int UserDataClient::sendDataIfReady() { // Get current date/time, format is YYYY-MM-DD.HH:mm:ss const std::string currentDateTime() { - time_t now = time(0); - struct tm tstruct; - char buf[80]; - tstruct = *localtime(&now); + time_t now = time(0); + struct tm tstruct; + char buf[80]; + tstruct = *localtime(&now); - strftime(buf, sizeof(buf), "%Y/%m/%d, %X", &tstruct); - return buf; + strftime(buf, sizeof(buf), "%Y/%m/%d, %X", &tstruct); + return buf; +} + +} } \ No newline at end of file diff --git a/test/cpp/qps/user_data_client.h b/test/cpp/qps/user_data_client.h index ec3dcb5d44b..0186ec90712 100644 --- a/test/cpp/qps/user_data_client.h +++ b/test/cpp/qps/user_data_client.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -44,27 +45,22 @@ #include #include "test/cpp/qps/user_data.grpc.pb.h" -using grpc::ChannelArguments; -using grpc::ChannelInterface; -using grpc::ClientContext; -using grpc::Status; -using UserData::UserDataTransfer; -using UserData::Metrics; -using UserData::SingleUserRecordRequest; -using UserData::SingleUserRecordReply; + +namespace grpc{ +namespace testing { class UserDataClient { - public: +public: UserDataClient(std::shared_ptr channel) : stub_(UserDataTransfer::NewStub(channel)) {} ~UserDataClient() {} - void setAccessToken(std::string access_token); + void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig); void setQPS(double QPS); - void setQPSPerCore(double qpsPerCore); + void setQPSPerCore(double QPSPerCore); void setLatencies(double percentileLatency50, double percentileLatency90, double percentileLatency95, double percentileLatency99, double percentileLatency99Point9); @@ -72,17 +68,26 @@ class UserDataClient { void setTimes(double serverSystemTime, double serverUserTime, double clientSystemTime, double clientUserTime); - int sendDataIfReady(); + int sendData(std::string access_token, std::string test_name); - private: +private: std::unique_ptr stub_; - std::string access_token_; - double QPS_; - double percentileLatency50_; - double percentileLatency90_; - double percentileLatency95_; - double percentileLatency99_; - double percentileLatency99Point9_; - bool qpsSet = false; - bool latenciesSet = false; -}; \ No newline at end of file + ClientConfig clientConfig_; + ServerConfig serverConfig_; + double QPS_ = DBL_MIN; + double QPSPerCore_ = DBL_MIN; + double percentileLatency50_ = DBL_MIN; + double percentileLatency90_ = DBL_MIN; + double percentileLatency95_ = DBL_MIN; + double percentileLatency99_ = DBL_MIN; + double percentileLatency99Point9_ = DBL_MIN; + double serverSystemTime_ = DBL_MIN; + double serverUserTime_ = DBL_MIN; + double clientSystemTime_ = DBL_MIN; + double clientUserTime_ = DBL_MIN; +}; + +} //namespace testing +} //namespace grpc + + diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc index b68822436db..6b573abb50b 100644 --- a/test/cpp/util/benchmark_config.cc +++ b/test/cpp/util/benchmark_config.cc @@ -39,6 +39,8 @@ DEFINE_bool(enable_log_reporter, true, DEFINE_string(access_token, "", "Authorizing JSON string for leaderboard"); +DEFINE_string(test_name, "", "Name of the test being executed"); + // In some distros, gflags is in the namespace google, and in some others, // in gflags. This hack is enabling us to find both. namespace google {} @@ -61,7 +63,7 @@ static std::shared_ptr InitBenchmarkReporters() { } if(!FLAGS_access_token.empty()) composite_reporter->add( - std::unique_ptr(new UserDatabaseReporter("UserDataReporter", FLAGS_access_token))); + std::unique_ptr(new UserDatabaseReporter("UserDataReporter", FLAGS_access_token, FLAGS_test_name))); return std::shared_ptr(composite_reporter); } From ef3ca1b3bf0f3f8f05bfc47f84ace7be0d0052ee Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 8 Jun 2015 14:55:24 -0700 Subject: [PATCH 03/63] stable --- test/cpp/qps/report.cc | 13 ++++++------- test/cpp/qps/report.h | 3 +-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 75e0e3ebd8d..5642ec8d2d5 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -124,7 +124,7 @@ UserDataClient userDataClient(grpc::CreateChannel("localhost:50052", grpc::Insec //Leaderboard Reported implementation. void UserDatabaseReporter::ReportQPS(const ScenarioResult& result) const { - double qps = result.latencies.Count() / + auto qps = result.latencies.Count() / average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); @@ -132,13 +132,12 @@ void UserDatabaseReporter::ReportQPS(const ScenarioResult& result) const { userDataClient.setConfigs(result.client_config, result.server_config); } -void UserDatabaseReporter::ReportQPSPerCore(const ScenarioResult& result, - const ServerConfig& server_config) const { - double qps = result.latencies.Count() / - average(result.client_resources, - [](ResourceUsage u) { return u.wall_time; }); +void UserDatabaseReporter::ReportQPSPerCore(const ScenarioResult& result) const { + auto qps = result.latencies.Count() / + average(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); - double qpsPerCore = qps / server_config.threads(); + auto qpsPerCore = qps / result.server_config.threads(); userDataClient.setQPSPerCore(qpsPerCore); userDataClient.setConfigs(result.client_config, result.server_config); diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 48da77039aa..ea09c385853 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -114,8 +114,7 @@ class UserDatabaseReporter : public Reporter { std::string access_token_; std::string test_name_; void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE; - void ReportQPSPerCore(const ScenarioResult& result, - const ServerConfig& config) const GRPC_OVERRIDE; + void ReportQPSPerCore(const ScenarioResult& result) const GRPC_OVERRIDE; void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE; void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; void Flush() const; From 0acc25ad15163008015e1ef6e8848ad312c6dc08 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 8 Jun 2015 15:53:55 -0700 Subject: [PATCH 04/63] removed shell script --- test/cpp/qps/run_authenticated_test.sh | 87 -------------------------- 1 file changed, 87 deletions(-) delete mode 100755 test/cpp/qps/run_authenticated_test.sh diff --git a/test/cpp/qps/run_authenticated_test.sh b/test/cpp/qps/run_authenticated_test.sh deleted file mode 100755 index 3cde6494142..00000000000 --- a/test/cpp/qps/run_authenticated_test.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/sh - -CLIENT_ID='1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' -CLIENT_SECRET='_HGHXg4DAA59r4w4x8p6ARzD' -GRANT_TYPE='http://oauth.net/grant_type/device/1.0' -ACCESS_TOKENS_DIR='/tmp/auth_lead_access_tokens' -AUTH_TOKEN_LINK='https://www.googleapis.com/oauth2/v3/token' -GOOGLE_ACCOUNTS_LINK='https://accounts.google.com/o/oauth2/device/code' -USER_INFO_LINK='https://www.googleapis.com/oauth2/v1/userinfo' -#Performs first time authentication -#Or re-authentication if refresh token expires -RE_AUTHENTICATE() { - INIT_AUTH_JSON=$(curl -s -d "client_id=$CLIENT_ID&scope=email profile" $GOOGLE_ACCOUNTS_LINK) - - USER_CODE=$(echo $INIT_AUTH_JSON | jq .user_code | sed -e 's/^"//' -e 's/"$//') - echo 'Please use the following user code in the browser:' $USER_CODE - echo - - VERIFICATION_URL=$(echo $INIT_AUTH_JSON | jq '.verification_url' | sed -e 's/^"//' -e 's/"$//') - echo 'Verification URL:' $VERIFICATION_URL - echo - - xdg-open $VERIFICATION_URL - - DEVICE_CODE=$(echo $INIT_AUTH_JSON | jq '.device_code' | sed -e 's/^"//' -e 's/"$//') - INTERVAL=$(echo $INIT_AUTH_JSON | jq '.interval' | sed -e 's/^"//' -e 's/"$//') - - AUTH_JSON=$(curl -s -d "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&code=$DEVICE_CODE&grant_type=$GRANT_TYPE" $AUTH_TOKEN_LINK) - ACCESS_TOKEN=$(echo $AUTH_JSON | jq '.access_token' | sed -e 's/^"//' -e 's/"$//') - - while [ $ACCESS_TOKEN == 'null' ] - do - sleep $INTERVAL - AUTH_JSON=$(curl -s -d "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&code=$DEVICE_CODE&grant_type=$GRANT_TYPE" $AUTH_TOKEN_LINK) - ACCESS_TOKEN=$(echo $AUTH_JSON | jq '.access_token' | sed -e 's/^"//' -e 's/"$//') - done - - USER_DETAILS=$(curl -s $USER_INFO_LINK?access_token=$ACCESS_TOKEN) - USER_ID=$(echo $USER_DETAILS | jq '.email' | sed -e 's/^"//' -e 's/"$//' | awk -F"@" '{print $1}' | sed -e 's/\.//g' | awk '{print tolower($0)}') - echo $AUTH_JSON > $ACCESS_TOKENS_DIR/$USER_ID -} - -#Use existing access token -USE_ACCESS_TOKEN() { - ACCESS_TOKEN=$(jq '.access_token' $ACCESS_TOKENS_DIR/$USER_ID | sed -e 's/^"//' -e 's/"$//') - - USER_DETAILS=$(curl -s $USER_INFO_LINK?access_token=$ACCESS_TOKEN) - - ID=$(echo $USER_DETAILS | jq '.id' | sed -e 's/^"//' -e 's/"$//') - - if [ $ID == 'null' ]; then - REFRESH_ACCESS_TOKEN - fi -} - -#Obtain new access token using refresh token -REFRESH_ACCESS_TOKEN() { - REFRESH_TOKEN=$(jq '.refresh_token' $ACCESS_TOKENS_DIR/$USER_ID | sed -e 's/^"//' -e 's/"$//') - if [ $REFRESH_TOKEN == 'null' ]; then - RE_AUTHENTICATE - else - REFRESH_JSON=$(curl -s -d "refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&grant_type=refresh_token" $AUTH_TOKEN_LINK) - - ACCESS_TOKEN=$(echo $REFRESH_JSON | jq '.access_token') - if [ $ACCESS_TOKEN == 'null' ]; then - RE_AUTHENTICATE - else - NEW_AUTH_JSON=$(jq ".access_token=$ACCESS_TOKEN" $ACCESS_TOKENS_DIR/$USER_ID) - echo $NEW_AUTH_JSON > $ACCESS_TOKENS_DIR/$USER_ID - fi - fi -} - -#create directory to store tokens, if not already present -[ ! -d $ACCESS_TOKENS_DIR ] && mkdir $ACCESS_TOKENS_DIR - -#Convert user entered email id to unique string by converting to splitting on '@' symbol, if present, -#removing '.'s and converting to lowercase -USER_ID=$(echo $2 | awk -F"@" '{print $1}' | sed -e 's/\.//g' | awk '{print tolower($0)}') - -if [ -s $ACCESS_TOKENS_DIR/$USER_ID ]; then - USE_ACCESS_TOKEN -else - RE_AUTHENTICATE -fi - -./$1 --access_token=$ACCESS_TOKEN \ No newline at end of file From f4dafb5654c8cf49a20fe4ec666d3f838dc90fef Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Wed, 10 Jun 2015 15:10:35 -0700 Subject: [PATCH 05/63] updated configs sending --- test/cpp/qps/data_script.sh | 7 +++++++ test/cpp/qps/report.cc | 4 ++-- test/cpp/qps/report.h | 9 +++++---- test/cpp/qps/run_auth_test.py | 6 +++++- test/cpp/qps/user_data.proto | 14 ++++++++------ test/cpp/qps/user_data_client.cc | 20 +++++--------------- test/cpp/qps/user_data_client.h | 2 +- test/cpp/util/benchmark_config.cc | 4 +++- 8 files changed, 36 insertions(+), 30 deletions(-) create mode 100644 test/cpp/qps/data_script.sh mode change 100644 => 100755 test/cpp/qps/run_auth_test.py diff --git a/test/cpp/qps/data_script.sh b/test/cpp/qps/data_script.sh new file mode 100644 index 00000000000..51d840f8d80 --- /dev/null +++ b/test/cpp/qps/data_script.sh @@ -0,0 +1,7 @@ +while [ true ] +do + python run_auth_test.py ../../../bins/opt/async_streaming_ping_pong_test sidrakesh@google.com + python run_auth_test.py ../../../bins/opt/async_unary_ping_pong_test sidrakesh@google.com + python run_auth_test.py ../../../bins/opt/sync_streaming_ping_pong_test sidrakesh@google.com + python run_auth_test.py ../../../bins/opt/sync_unary_ping_pong_test sidrakesh@google.com +done \ No newline at end of file diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 5642ec8d2d5..83860ab9cee 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -175,8 +175,8 @@ void UserDatabaseReporter::ReportTimes(const ScenarioResult& result) const { userDataClient.setConfigs(result.client_config, result.server_config); } -void UserDatabaseReporter::Flush() const { - int userDataState = userDataClient.sendData(access_token_, test_name_); +void UserDatabaseReporter::SendData() const { + int userDataState = userDataClient.sendData(access_token_, test_name_, sys_info_); switch(userDataState) { case 1: diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index ea09c385853..432ed5ddf2b 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -106,18 +106,19 @@ class GprLogReporter : public Reporter { /** Reporter for client leaderboard. */ class UserDatabaseReporter : public Reporter { public: - UserDatabaseReporter(const string& name, const string& access_token, - const string& test_name) : Reporter(name), access_token_(access_token), test_name_(test_name) {} - ~UserDatabaseReporter() { Flush(); }; + UserDatabaseReporter(const string& name, const string& access_token, const string& test_name, const string& sys_info) + : Reporter(name), access_token_(access_token), test_name_(test_name), sys_info_(sys_info) {} + ~UserDatabaseReporter() { SendData(); }; private: std::string access_token_; std::string test_name_; + std::string sys_info_; void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE; void ReportQPSPerCore(const ScenarioResult& result) const GRPC_OVERRIDE; void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE; void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; - void Flush() const; + void SendData() const; }; } // namespace testing diff --git a/test/cpp/qps/run_auth_test.py b/test/cpp/qps/run_auth_test.py old mode 100644 new mode 100755 index 3ed2a7c9804..75bdab22a4a --- a/test/cpp/qps/run_auth_test.py +++ b/test/cpp/qps/run_auth_test.py @@ -129,7 +129,11 @@ def main(): accessToken = reauthenticate() testName = sys.argv[1].split('/')[-1] - subprocess.call([sys.argv[1], '--access_token='+accessToken, '--test_name='+testName]) + sysInfo = os.popen('lscpu').readlines() + try: + subprocess.call([sys.argv[1], '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]')]) + except OSError: + print 'Could not execute the test, please check test path' if __name__ == "__main__": main() \ No newline at end of file diff --git a/test/cpp/qps/user_data.proto b/test/cpp/qps/user_data.proto index 27d0049cf07..947ccce60b6 100644 --- a/test/cpp/qps/user_data.proto +++ b/test/cpp/qps/user_data.proto @@ -61,9 +61,10 @@ message Metrics { message DataDetails { optional string timestamp = 1; optional string test_name = 2; - optional Metrics metrics = 3; - optional ClientConfig client_config = 4; - optional ServerConfig server_config = 5; + optional string sys_info = 3; + optional Metrics metrics = 4; + optional ClientConfig client_config = 5; + optional ServerConfig server_config = 6; } //User details @@ -91,9 +92,10 @@ message SingleUserDetails { message SingleUserRecordRequest { optional string access_token = 1; optional string test_name = 2; - optional Metrics metrics = 3; - optional ClientConfig client_config = 4; - optional ServerConfig server_config = 5; + optional string sys_info = 3; + optional Metrics metrics = 4; + optional ClientConfig client_config = 5; + optional ServerConfig server_config = 6; } //Reply to request for storing single user's data diff --git a/test/cpp/qps/user_data_client.cc b/test/cpp/qps/user_data_client.cc index 50b74276d2e..f56b79ee4a6 100644 --- a/test/cpp/qps/user_data_client.cc +++ b/test/cpp/qps/user_data_client.cc @@ -66,11 +66,13 @@ void UserDataClient::setTimes(double serverSystemTime, double serverUserTime, clientUserTime_ = clientUserTime; } -int UserDataClient::sendData(std::string access_token, std::string test_name) { +int UserDataClient::sendData(std::string access_token, std::string test_name, std::string sys_info) { SingleUserRecordRequest singleUserRecordRequest; singleUserRecordRequest.set_access_token(access_token); singleUserRecordRequest.set_test_name(test_name); + singleUserRecordRequest.set_sys_info(sys_info); + *(singleUserRecordRequest.mutable_client_config()) = clientConfig_; *(singleUserRecordRequest.mutable_server_config()) = serverConfig_; @@ -98,17 +100,5 @@ int UserDataClient::sendData(std::string access_token, std::string test_name) { return -1; } } - -// Get current date/time, format is YYYY-MM-DD.HH:mm:ss -const std::string currentDateTime() { - time_t now = time(0); - struct tm tstruct; - char buf[80]; - tstruct = *localtime(&now); - - strftime(buf, sizeof(buf), "%Y/%m/%d, %X", &tstruct); - return buf; -} - -} -} \ No newline at end of file +} //testing +} //grpc \ No newline at end of file diff --git a/test/cpp/qps/user_data_client.h b/test/cpp/qps/user_data_client.h index 0186ec90712..c99b7e9b3a3 100644 --- a/test/cpp/qps/user_data_client.h +++ b/test/cpp/qps/user_data_client.h @@ -68,7 +68,7 @@ public: void setTimes(double serverSystemTime, double serverUserTime, double clientSystemTime, double clientUserTime); - int sendData(std::string access_token, std::string test_name); + int sendData(std::string access_token, std::string test_name, std::string sys_info); private: std::unique_ptr stub_; diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc index 6b573abb50b..82663b6a013 100644 --- a/test/cpp/util/benchmark_config.cc +++ b/test/cpp/util/benchmark_config.cc @@ -41,6 +41,8 @@ DEFINE_string(access_token, "", "Authorizing JSON string for leaderboard"); DEFINE_string(test_name, "", "Name of the test being executed"); +DEFINE_string(sys_info, "", "System information"); + // In some distros, gflags is in the namespace google, and in some others, // in gflags. This hack is enabling us to find both. namespace google {} @@ -63,7 +65,7 @@ static std::shared_ptr InitBenchmarkReporters() { } if(!FLAGS_access_token.empty()) composite_reporter->add( - std::unique_ptr(new UserDatabaseReporter("UserDataReporter", FLAGS_access_token, FLAGS_test_name))); + std::unique_ptr(new UserDatabaseReporter("UserDataReporter", FLAGS_access_token, FLAGS_test_name, FLAGS_sys_info))); return std::shared_ptr(composite_reporter); } From b2b5b9a06946ca5b9cad759f7d5731b255b25e90 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Wed, 10 Jun 2015 15:12:34 -0700 Subject: [PATCH 06/63] removed data adding script --- test/cpp/qps/data_script.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 test/cpp/qps/data_script.sh diff --git a/test/cpp/qps/data_script.sh b/test/cpp/qps/data_script.sh deleted file mode 100644 index 51d840f8d80..00000000000 --- a/test/cpp/qps/data_script.sh +++ /dev/null @@ -1,7 +0,0 @@ -while [ true ] -do - python run_auth_test.py ../../../bins/opt/async_streaming_ping_pong_test sidrakesh@google.com - python run_auth_test.py ../../../bins/opt/async_unary_ping_pong_test sidrakesh@google.com - python run_auth_test.py ../../../bins/opt/sync_streaming_ping_pong_test sidrakesh@google.com - python run_auth_test.py ../../../bins/opt/sync_unary_ping_pong_test sidrakesh@google.com -done \ No newline at end of file From 7b55362e0fcb55f42240c1164678709e50504b60 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Thu, 11 Jun 2015 20:59:14 -0700 Subject: [PATCH 07/63] updated auth script --- test/cpp/qps/run_auth_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/cpp/qps/run_auth_test.py b/test/cpp/qps/run_auth_test.py index 75bdab22a4a..06b064cad0c 100755 --- a/test/cpp/qps/run_auth_test.py +++ b/test/cpp/qps/run_auth_test.py @@ -115,7 +115,11 @@ def main(): if not os.path.exists(ACCESS_TOKENS_DIR): os.makedirs(ACCESS_TOKENS_DIR) - email = sys.argv[2] + if len(sys.argv) > 2: + email = sys.argv[2] + else: + email = raw_input('Enter your e-mail id: ') + email = email.split('@')[0].lower() userId = re.sub('[.]', '', email) From d0010ae439406e4118a1ca1788276d2c1f6fe9e5 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 15 Jun 2015 17:02:51 -0700 Subject: [PATCH 08/63] added comments --- test/cpp/qps/data_script.sh | 7 ++ test/cpp/qps/report.cc | 2 + test/cpp/qps/run_auth_test.py | 138 ++++++++++++++++++++++++++----- test/cpp/qps/user_data_client.cc | 16 +++- test/cpp/qps/user_data_client.h | 7 ++ 5 files changed, 146 insertions(+), 24 deletions(-) create mode 100644 test/cpp/qps/data_script.sh diff --git a/test/cpp/qps/data_script.sh b/test/cpp/qps/data_script.sh new file mode 100644 index 00000000000..51d840f8d80 --- /dev/null +++ b/test/cpp/qps/data_script.sh @@ -0,0 +1,7 @@ +while [ true ] +do + python run_auth_test.py ../../../bins/opt/async_streaming_ping_pong_test sidrakesh@google.com + python run_auth_test.py ../../../bins/opt/async_unary_ping_pong_test sidrakesh@google.com + python run_auth_test.py ../../../bins/opt/sync_streaming_ping_pong_test sidrakesh@google.com + python run_auth_test.py ../../../bins/opt/sync_unary_ping_pong_test sidrakesh@google.com +done \ No newline at end of file diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 83860ab9cee..91fe9524cb4 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -176,8 +176,10 @@ void UserDatabaseReporter::ReportTimes(const ScenarioResult& result) const { } void UserDatabaseReporter::SendData() const { + //send data to performance database int userDataState = userDataClient.sendData(access_token_, test_name_, sys_info_); + //check state of data sending switch(userDataState) { case 1: gpr_log(GPR_INFO, "Data sent to user database successfully"); diff --git a/test/cpp/qps/run_auth_test.py b/test/cpp/qps/run_auth_test.py index 06b064cad0c..c327a505b6f 100755 --- a/test/cpp/qps/run_auth_test.py +++ b/test/cpp/qps/run_auth_test.py @@ -1,3 +1,33 @@ +# +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# #!/usr/bin/python import os @@ -8,6 +38,7 @@ import urllib import json import time import subprocess +import fnmatch CLIENT_ID = '1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' CLIENT_SECRET = '_HGHXg4DAA59r4w4x8p6ARzD' @@ -17,6 +48,7 @@ AUTH_TOKEN_LINK = 'https://www.googleapis.com/oauth2/v3/token' GOOGLE_ACCOUNTS_LINK = 'https://accounts.google.com/o/oauth2/device/code' USER_INFO_LINK = 'https://www.googleapis.com/oauth2/v1/userinfo' +# Fetches JSON reply object, given a url and parameters def fetchJSON(url, paramDict): if len(paramDict) == 0: req = urllib2.Request(url) @@ -33,6 +65,7 @@ def fetchJSON(url, paramDict): return result +# Fetch user info; used to check if access token is valid def getUserInfo(accessToken): url = USER_INFO_LINK + '?access_token=' + accessToken paramDict = {} @@ -41,6 +74,7 @@ def getUserInfo(accessToken): return data +# Returns true if stored access token is valid def isAccessTokenValid(accessToken): data = getUserInfo(accessToken); @@ -49,32 +83,47 @@ def isAccessTokenValid(accessToken): else: return False +# Returns user id given a working access token def getUserId(accessToken): data = getUserInfo(accessToken) email = data['email'] - email = email.split('@')[0].lower() - userId = re.sub('[.]', '', email) + userId = getUserIdFromEmail(email) return userId +# Extracts a unique user id from an email address +def getUserIdFromEmail(email): + email = email.split('@')[0].lower() # take username and convert to lower case + userId = re.sub('[.]', '', email) # remove periods + + return userId + +# Use an existing access token def useAccessToken(userTokFile): with open(userTokFile, "r") as data_file: - data = json.load(data_file) + data = json.load(data_file) # load JSON data from file accessToken = data["access_token"] + # If access token has gone stale, refresh it if not isAccessTokenValid(accessToken): return refreshAccessToken(data["refresh_token"], userTokFile) return accessToken +# refresh stale access token def refreshAccessToken(refreshToken, userTokFile): + # Parameters for request paramDict = {'refresh_token':refreshToken, 'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'grant_type':'refresh_token'} + # Fetch reply to request JSONBody = fetchJSON(AUTH_TOKEN_LINK, paramDict) data = json.loads(JSONBody) + if not 'access_token' in data: + # Refresh token has gone stale, re-authentication required return reauthenticate() else: + # write fresh access token to tokens file tokenData = {} with open(userTokFile, "r") as data_file: @@ -83,61 +132,108 @@ def refreshAccessToken(refreshToken, userTokFile): with open(userTokFile, "w") as data_file: tokenData['access_token'] = data['access_token'] json.dump(tokenData, data_file) - + + # return fresh access token return data['access_token'] def reauthenticate(): + # Request parameters paramDict = {'client_id':CLIENT_ID, 'scope':'email profile'} JSONBody = fetchJSON(GOOGLE_ACCOUNTS_LINK, paramDict) data = json.loads(JSONBody) print 'User authorization required\n' - print 'Please use the following code in you browser: ', data['user_code'] - print 'Verification URL: ', data['verification_url'] + print 'Please use the following code in you browser: ', data['user_code'] # Code to be entered by user in browser + print 'Verification URL: ', data['verification_url'] # Authentication link print '\nAwaiting user authorization. May take a few more seconds after authorizing...\n' authData = {} while not 'access_token' in authData: + # Request parameters authDict = {'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'code':data['device_code'], 'grant_type':GRANT_TYPE} JSONBody = fetchJSON(AUTH_TOKEN_LINK, authDict) authData = json.loads(JSONBody) + # If server pinged too quickly, will get slowdown message; need to wait for specified interval time.sleep(data['interval']) + # File to write tokens newUserTokFile = ACCESS_TOKENS_DIR + '/' + getUserId(authData['access_token']) + # Write tokens to file with open(newUserTokFile, "w") as data_file: json.dump(authData, data_file) + # return working access token return authData['access_token'] -def main(): - if not os.path.exists(ACCESS_TOKENS_DIR): - os.makedirs(ACCESS_TOKENS_DIR) - - if len(sys.argv) > 2: - email = sys.argv[2] - else: - email = raw_input('Enter your e-mail id: ') - - email = email.split('@')[0].lower() - userId = re.sub('[.]', '', email) +# Fetch a working access token given user entered email id; authntication may be required +def getAccessToken(email): + # Get unique user id from email address + userId = getUserIdFromEmail(email) + # Token file userTokFile = ACCESS_TOKENS_DIR + '/' + userId accessToken = '' if os.path.exists(userTokFile): + # File containing access token exists; unless refresh token has expired, user authentication will not be required accessToken = useAccessToken(userTokFile) else: + # User authentication required accessToken = reauthenticate() - testName = sys.argv[1].split('/')[-1] - sysInfo = os.popen('lscpu').readlines() + return accessToken + +# If user has not entered full path to test, recursively searches for given test in parent folders +def findTestPath(test): + # If user entered full path to test, return it + if(os.path.isfile(test)): + return test + + testName = test.split('/')[-1] # Extract just test name + testPath = '' + + # Search for test + for root, dirnames, filenames in os.walk('../../../'): + for fileName in fnmatch.filter(filenames, '*'+testName): + testPath = os.path.join(root, fileName) + + return testPath + +def main(): + # If tokens directory does not exist, creates it + if not os.path.exists(ACCESS_TOKENS_DIR): + os.makedirs(ACCESS_TOKENS_DIR) + + if len(sys.argv) > 1: + test = sys.argv[1] + else: + test = raw_input('Enter the test path/name: ') + + if len(sys.argv) > 2: + email = sys.argv[2] + else: + email = raw_input('Enter your e-mail id: ') + + try: + # Fetch working access token + accessToken = getAccessToken(email) + except Exception, e: + print 'Error in authentication' try: - subprocess.call([sys.argv[1], '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]')]) + testPath = findTestPath(test) # Get path to test + testName = testPath.split('/')[-1] # Get test name + + # Fetch system information + sysInfo = os.popen('lscpu').readlines() + + # Run the test + subprocess.call([testPath, '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]')]) except OSError: - print 'Could not execute the test, please check test path' + print 'Could not execute the test, please check test name' + if __name__ == "__main__": main() \ No newline at end of file diff --git a/test/cpp/qps/user_data_client.cc b/test/cpp/qps/user_data_client.cc index f56b79ee4a6..a8ced7559d7 100644 --- a/test/cpp/qps/user_data_client.cc +++ b/test/cpp/qps/user_data_client.cc @@ -36,19 +36,23 @@ namespace grpc { namespace testing { +//sets the client and server config information void UserDataClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) { clientConfig_ = clientConfig; serverConfig_ = serverConfig; } +//sets the QPS void UserDataClient::setQPS(double QPS) { QPS_ = QPS; } +//sets the QPS per core void UserDataClient::setQPSPerCore(double QPSPerCore) { QPSPerCore_ = QPSPerCore; } +//sets the 50th, 90th, 95th, 99th and 99.9th percentile latency void UserDataClient::setLatencies(double percentileLatency50, double percentileLatency90, double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) { percentileLatency50_ = percentileLatency50; @@ -58,6 +62,7 @@ void UserDataClient::setLatencies(double percentileLatency50, double percentileL percentileLatency99Point9_ = percentileLatency99Point9; } +//sets the server and client, user and system times void UserDataClient::setTimes(double serverSystemTime, double serverUserTime, double clientSystemTime, double clientUserTime) { serverSystemTime_ = serverSystemTime; @@ -66,18 +71,23 @@ void UserDataClient::setTimes(double serverSystemTime, double serverUserTime, clientUserTime_ = clientUserTime; } +//sends the data to the performancew database server int UserDataClient::sendData(std::string access_token, std::string test_name, std::string sys_info) { - + //Data record request object SingleUserRecordRequest singleUserRecordRequest; + + //setting access token, name of the test and the system information singleUserRecordRequest.set_access_token(access_token); singleUserRecordRequest.set_test_name(test_name); singleUserRecordRequest.set_sys_info(sys_info); + //setting configs *(singleUserRecordRequest.mutable_client_config()) = clientConfig_; *(singleUserRecordRequest.mutable_server_config()) = serverConfig_; Metrics* metrics = singleUserRecordRequest.mutable_metrics(); + //setting metrcs in data record request if(QPS_ != DBL_MIN) metrics->set_qps(QPS_); if(QPSPerCore_ != DBL_MIN) metrics->set_qps_per_core(QPSPerCore_); if(percentileLatency50_ != DBL_MIN) metrics->set_perc_lat_50(percentileLatency50_); @@ -95,9 +105,9 @@ int UserDataClient::sendData(std::string access_token, std::string test_name, st Status status = stub_->RecordSingleClientData(&context, singleUserRecordRequest, &singleUserRecordReply); if (status.IsOk()) { - return 1; + return 1; //data sent to database successfully } else { - return -1; + return -1; //error in data sending } } } //testing diff --git a/test/cpp/qps/user_data_client.h b/test/cpp/qps/user_data_client.h index c99b7e9b3a3..c2e07ef5cd9 100644 --- a/test/cpp/qps/user_data_client.h +++ b/test/cpp/qps/user_data_client.h @@ -49,6 +49,7 @@ namespace grpc{ namespace testing { +//Manages data sending to performance database server class UserDataClient { public: UserDataClient(std::shared_ptr channel) @@ -56,18 +57,24 @@ public: ~UserDataClient() {} + //sets the client and server config information void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig); + //sets the QPS void setQPS(double QPS); + //sets the QPS per core void setQPSPerCore(double QPSPerCore); + //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency void setLatencies(double percentileLatency50, double percentileLatency90, double percentileLatency95, double percentileLatency99, double percentileLatency99Point9); + //sets the server and client, user and system times void setTimes(double serverSystemTime, double serverUserTime, double clientSystemTime, double clientUserTime); + //sends the data to the performancew database server int sendData(std::string access_token, std::string test_name, std::string sys_info); private: From 08e8bab4b81f8c2970b17ef8377e30d766eb146b Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 15 Jun 2015 17:04:09 -0700 Subject: [PATCH 09/63] removed data script --- test/cpp/qps/data_script.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 test/cpp/qps/data_script.sh diff --git a/test/cpp/qps/data_script.sh b/test/cpp/qps/data_script.sh deleted file mode 100644 index 51d840f8d80..00000000000 --- a/test/cpp/qps/data_script.sh +++ /dev/null @@ -1,7 +0,0 @@ -while [ true ] -do - python run_auth_test.py ../../../bins/opt/async_streaming_ping_pong_test sidrakesh@google.com - python run_auth_test.py ../../../bins/opt/async_unary_ping_pong_test sidrakesh@google.com - python run_auth_test.py ../../../bins/opt/sync_streaming_ping_pong_test sidrakesh@google.com - python run_auth_test.py ../../../bins/opt/sync_unary_ping_pong_test sidrakesh@google.com -done \ No newline at end of file From e2902dee317bdab3cca20501032d74ffdd1b6f55 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Wed, 17 Jun 2015 15:01:43 -0700 Subject: [PATCH 10/63] commented --- test/cpp/qps/report.cc | 1 + test/cpp/qps/run_auth_test.py | 47 ++++++++++++++++++++++++++----- test/cpp/util/benchmark_config.cc | 5 +++- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 91fe9524cb4..25cc13156a9 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -139,6 +139,7 @@ void UserDatabaseReporter::ReportQPSPerCore(const ScenarioResult& result) const auto qpsPerCore = qps / result.server_config.threads(); + userDataClient.setQPS(qps); userDataClient.setQPSPerCore(qpsPerCore); userDataClient.setConfigs(result.client_config, result.server_config); } diff --git a/test/cpp/qps/run_auth_test.py b/test/cpp/qps/run_auth_test.py index c327a505b6f..39a4a0ae3fc 100755 --- a/test/cpp/qps/run_auth_test.py +++ b/test/cpp/qps/run_auth_test.py @@ -59,7 +59,7 @@ def fetchJSON(url, paramDict): try: response = urllib2.urlopen(req) result = response.read() - + except urllib2.HTTPError, error: result = error.read() @@ -71,7 +71,7 @@ def getUserInfo(accessToken): paramDict = {} JSONBody = fetchJSON(url, paramDict) data = json.loads(JSONBody) - + return data # Returns true if stored access token is valid @@ -197,11 +197,44 @@ def findTestPath(test): # Search for test for root, dirnames, filenames in os.walk('../../../'): - for fileName in fnmatch.filter(filenames, '*'+testName): - testPath = os.path.join(root, fileName) + for fileName in fnmatch.filter(filenames, testName): + testPath = os.path.join(root, fileName) return testPath +def getSysInfo(): + # Fetch system information + sysInfo = os.popen('lscpu').readlines() + + NICs = os.popen('ifconfig | cut -c1-8 | sed \'/^\s*$/d\' | sort -u').readlines() + nicAddrs = os.popen('ifconfig | grep -oE "inet addr:([0-9]{1,3}\.){3}[0-9]{1,3}"').readlines() + + nicInfo = [] + + for i in range(0, len(NICs)): + NIC = NICs[i] + NIC = re.sub(r'[^\w]', '', NIC) + + ethtoolProcess = subprocess.Popen(["ethtool",NIC], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ethtoolResult = ethtoolProcess.communicate()[0] + + ethtoolResultList = ethtoolResult.split('\n\t') + for ethtoolString in ethtoolResultList: + if ethtoolString.startswith('Speed'): + ethtoolString = ethtoolString.split(':')[1] + ethtoolString = ethtoolString.replace('Mb/s',' Mbps') + nicInfo.append(NIC + ' speed: ' + ethtoolString + '\n') + nicInfo.append(NIC + ' inet address: ' + nicAddrs[i].split(':')[1]) + + print 'Obtaining network info....' + tcp_rr_rate = str(os.popen('netperf -t TCP_RR -v 0').readlines()[1]) + print 'Network info obtained' + + nicInfo.append('TCP RR Transmission Rate per sec: ' + tcp_rr_rate + '\n') + sysInfo = sysInfo + nicInfo + + return sysInfo + def main(): # If tokens directory does not exist, creates it if not os.path.exists(ACCESS_TOKENS_DIR): @@ -227,11 +260,11 @@ def main(): testPath = findTestPath(test) # Get path to test testName = testPath.split('/')[-1] # Get test name - # Fetch system information - sysInfo = os.popen('lscpu').readlines() + sysInfo = getSysInfo() + print '\nBeginning test:\n' # Run the test - subprocess.call([testPath, '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]')]) + subprocess.call([testPath, '--report_metrics_db=true', '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]')]) except OSError: print 'Could not execute the test, please check test name' diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc index 82663b6a013..250bb7ac8e3 100644 --- a/test/cpp/util/benchmark_config.cc +++ b/test/cpp/util/benchmark_config.cc @@ -37,6 +37,8 @@ DEFINE_bool(enable_log_reporter, true, "Enable reporting of benchmark results through GprLog"); +DEFINE_bool(report_metrics_db, false, "True if metrics to be reported to performance database"); + DEFINE_string(access_token, "", "Authorizing JSON string for leaderboard"); DEFINE_string(test_name, "", "Name of the test being executed"); @@ -63,9 +65,10 @@ static std::shared_ptr InitBenchmarkReporters() { composite_reporter->add( std::unique_ptr(new GprLogReporter("LogReporter"))); } - if(!FLAGS_access_token.empty()) + if(FLAGS_report_metrics_db) { composite_reporter->add( std::unique_ptr(new UserDatabaseReporter("UserDataReporter", FLAGS_access_token, FLAGS_test_name, FLAGS_sys_info))); + } return std::shared_ptr(composite_reporter); } From 3df5c44c84c07c793da1dc9b68bc35ece4268d5e Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 17 Jun 2015 18:31:58 -0700 Subject: [PATCH 11/63] Introduced function to create byte buffers from the output of a byte buffer reader. --- include/grpc/byte_buffer.h | 4 ++++ src/core/surface/byte_buffer.c | 14 ++++++++++++ test/core/surface/byte_buffer_reader_test.c | 25 +++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/include/grpc/byte_buffer.h b/include/grpc/byte_buffer.h index 6d08474d8c1..1b51569f24a 100644 --- a/include/grpc/byte_buffer.h +++ b/include/grpc/byte_buffer.h @@ -103,6 +103,10 @@ void grpc_byte_buffer_reader_destroy(grpc_byte_buffer_reader *reader); int grpc_byte_buffer_reader_next(grpc_byte_buffer_reader *reader, gpr_slice *slice); +/** Returns a RAW byte buffer instance from the output of \a reader. */ +grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( + grpc_byte_buffer_reader *reader); + #ifdef __cplusplus } #endif diff --git a/src/core/surface/byte_buffer.c b/src/core/surface/byte_buffer.c index 4817e004549..a930949f2d4 100644 --- a/src/core/surface/byte_buffer.c +++ b/src/core/surface/byte_buffer.c @@ -55,6 +55,20 @@ grpc_byte_buffer *grpc_raw_compressed_byte_buffer_create( return bb; } +grpc_byte_buffer *grpc_raw_byte_buffer_from_reader( + grpc_byte_buffer_reader *reader) { + grpc_byte_buffer *bb = malloc(sizeof(grpc_byte_buffer)); + gpr_slice slice; + bb->type = GRPC_BB_RAW; + bb->data.raw.compression = GRPC_COMPRESS_NONE; + gpr_slice_buffer_init(&bb->data.raw.slice_buffer); + + while (grpc_byte_buffer_reader_next(reader, &slice)) { + gpr_slice_buffer_add(&bb->data.raw.slice_buffer, slice); + } + return bb; +} + grpc_byte_buffer *grpc_byte_buffer_copy(grpc_byte_buffer *bb) { switch (bb->type) { case GRPC_BB_RAW: diff --git a/test/core/surface/byte_buffer_reader_test.c b/test/core/surface/byte_buffer_reader_test.c index 87a2cd7a43b..2eb354a80aa 100644 --- a/test/core/surface/byte_buffer_reader_test.c +++ b/test/core/surface/byte_buffer_reader_test.c @@ -160,6 +160,30 @@ static void test_read_deflate_compressed_slice(void) { read_compressed_slice(GRPC_COMPRESS_DEFLATE, INPUT_SIZE); } +static void test_byte_buffer_from_reader(void) { + gpr_slice slice; + grpc_byte_buffer *buffer, *buffer_from_reader; + grpc_byte_buffer_reader reader; + + LOG_TEST("test_byte_buffer_from_reader"); + slice = gpr_slice_malloc(4); + memcpy(GPR_SLICE_START_PTR(slice), "test", 4); + buffer = grpc_raw_byte_buffer_create(&slice, 1); + gpr_slice_unref(slice); + grpc_byte_buffer_reader_init(&reader, buffer); + + buffer_from_reader = grpc_raw_byte_buffer_from_reader(&reader); + GPR_ASSERT(buffer->type == buffer_from_reader->type); + GPR_ASSERT(buffer_from_reader->data.raw.compression == GRPC_COMPRESS_NONE); + GPR_ASSERT(buffer_from_reader->data.raw.slice_buffer.count == 1); + GPR_ASSERT(memcmp(GPR_SLICE_START_PTR( + buffer_from_reader->data.raw.slice_buffer.slices[0]), + "test", 4) == 0); + + grpc_byte_buffer_destroy(buffer); + grpc_byte_buffer_destroy(buffer_from_reader); +} + int main(int argc, char **argv) { grpc_test_init(argc, argv); test_read_one_slice(); @@ -167,6 +191,7 @@ int main(int argc, char **argv) { test_read_none_compressed_slice(); test_read_gzip_compressed_slice(); test_read_deflate_compressed_slice(); + test_byte_buffer_from_reader(); return 0; } From 929191b4be8d958c927499092cb5f963b11148eb Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Thu, 18 Jun 2015 09:13:53 -0700 Subject: [PATCH 12/63] edited test script --- test/cpp/qps/{run_auth_test.py => run_perf_db_test.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename test/cpp/qps/{run_auth_test.py => run_perf_db_test.py} (99%) diff --git a/test/cpp/qps/run_auth_test.py b/test/cpp/qps/run_perf_db_test.py similarity index 99% rename from test/cpp/qps/run_auth_test.py rename to test/cpp/qps/run_perf_db_test.py index 39a4a0ae3fc..b2ca033e7ec 100755 --- a/test/cpp/qps/run_auth_test.py +++ b/test/cpp/qps/run_perf_db_test.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # # Copyright 2015, Google Inc. # All rights reserved. @@ -28,7 +29,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -#!/usr/bin/python import os import sys From bfa67ddbd4a5515ae92bd336390a4f6b71bcb661 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Thu, 18 Jun 2015 10:37:41 -0700 Subject: [PATCH 13/63] changed access tokens folder position --- test/cpp/qps/run_perf_db_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/cpp/qps/run_perf_db_test.py b/test/cpp/qps/run_perf_db_test.py index b2ca033e7ec..7f840fa723d 100755 --- a/test/cpp/qps/run_perf_db_test.py +++ b/test/cpp/qps/run_perf_db_test.py @@ -43,7 +43,7 @@ import fnmatch CLIENT_ID = '1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' CLIENT_SECRET = '_HGHXg4DAA59r4w4x8p6ARzD' GRANT_TYPE = 'http://oauth.net/grant_type/device/1.0' -ACCESS_TOKENS_DIR = '/tmp/auth_lead_access_tokens' +ACCESS_TOKENS_DIR = '/usr/local/auth_access_tokens' AUTH_TOKEN_LINK = 'https://www.googleapis.com/oauth2/v3/token' GOOGLE_ACCOUNTS_LINK = 'https://accounts.google.com/o/oauth2/device/code' USER_INFO_LINK = 'https://www.googleapis.com/oauth2/v1/userinfo' @@ -223,7 +223,7 @@ def getSysInfo(): if ethtoolString.startswith('Speed'): ethtoolString = ethtoolString.split(':')[1] ethtoolString = ethtoolString.replace('Mb/s',' Mbps') - nicInfo.append(NIC + ' speed: ' + ethtoolString + '\n') + nicInfo.append('NIC ' + NIC + ' speed: ' + ethtoolString + '\n') nicInfo.append(NIC + ' inet address: ' + nicAddrs[i].split(':')[1]) print 'Obtaining network info....' @@ -238,7 +238,8 @@ def getSysInfo(): def main(): # If tokens directory does not exist, creates it if not os.path.exists(ACCESS_TOKENS_DIR): - os.makedirs(ACCESS_TOKENS_DIR) + subprocess.call(['sudo', 'mkdir', ACCESS_TOKENS_DIR]) + subprocess.call(['sudo', 'chmod', '777', ACCESS_TOKENS_DIR]) if len(sys.argv) > 1: test = sys.argv[1] From e74a458813fcdf8b35080e30fc1190edcd5b88be Mon Sep 17 00:00:00 2001 From: sidrakesh93 Date: Thu, 18 Jun 2015 11:22:06 -0700 Subject: [PATCH 14/63] Update report.cc --- test/cpp/qps/report.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 25cc13156a9..c524327ff03 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -122,7 +122,7 @@ void GprLogReporter::ReportTimes(const ScenarioResult& result) const { UserDataClient userDataClient(grpc::CreateChannel("localhost:50052", grpc::InsecureCredentials(), ChannelArguments())); -//Leaderboard Reported implementation. +//Performance database reporter implementation. void UserDatabaseReporter::ReportQPS(const ScenarioResult& result) const { auto qps = result.latencies.Count() / average(result.client_resources, From cc5857b624a0f99af1cdf9944213673080d0fe52 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Thu, 18 Jun 2015 16:45:55 -0700 Subject: [PATCH 15/63] Fixed hanging client declaration, server address can be passed now --- Makefile | 46 ++++++++--------- build.json | 10 ++-- .../qps/{user_data.proto => perf_db.proto} | 2 +- ...{user_data_client.cc => perf_db_client.cc} | 20 ++++---- .../{user_data_client.h => perf_db_client.h} | 51 ++++++++++--------- test/cpp/qps/report.cc | 41 ++++++++------- test/cpp/qps/report.h | 12 +++-- test/cpp/qps/run_perf_db_test.py | 4 +- test/cpp/util/benchmark_config.cc | 4 +- 9 files changed, 99 insertions(+), 91 deletions(-) rename test/cpp/qps/{user_data.proto => perf_db.proto} (99%) rename test/cpp/qps/{user_data_client.cc => perf_db_client.cc} (86%) rename test/cpp/qps/{user_data_client.h => perf_db_client.h} (71%) diff --git a/Makefile b/Makefile index ef7795c2902..a3f6803a3c6 100644 --- a/Makefile +++ b/Makefile @@ -2502,30 +2502,30 @@ $(GENDIR)/examples/pubsub/pubsub.grpc.pb.cc: examples/pubsub/pubsub.proto $(PROT endif ifeq ($(NO_PROTOC),true) -$(GENDIR)/test/cpp/qps/qpstest.pb.cc: protoc_dep_error -$(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc: protoc_dep_error +$(GENDIR)/test/cpp/qps/perf_db.pb.cc: protoc_dep_error +$(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc: protoc_dep_error else -$(GENDIR)/test/cpp/qps/qpstest.pb.cc: test/cpp/qps/qpstest.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) +$(GENDIR)/test/cpp/qps/perf_db.pb.cc: test/cpp/qps/perf_db.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(E) "[PROTOC] Generating protobuf CC file from $<" $(Q) mkdir -p `dirname $@` $(Q) $(PROTOC) --cpp_out=$(GENDIR) $< -$(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc: test/cpp/qps/qpstest.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) +$(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc: test/cpp/qps/perf_db.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(E) "[GRPC] Generating gRPC's protobuf service CC file from $<" $(Q) mkdir -p `dirname $@` $(Q) $(PROTOC) --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $< endif ifeq ($(NO_PROTOC),true) -$(GENDIR)/test/cpp/qps/user_data.pb.cc: protoc_dep_error -$(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc: protoc_dep_error +$(GENDIR)/test/cpp/qps/qpstest.pb.cc: protoc_dep_error +$(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc: protoc_dep_error else -$(GENDIR)/test/cpp/qps/user_data.pb.cc: test/cpp/qps/user_data.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) +$(GENDIR)/test/cpp/qps/qpstest.pb.cc: test/cpp/qps/qpstest.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(E) "[PROTOC] Generating protobuf CC file from $<" $(Q) mkdir -p `dirname $@` $(Q) $(PROTOC) --cpp_out=$(GENDIR) $< -$(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc: test/cpp/qps/user_data.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) +$(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc: test/cpp/qps/qpstest.proto $(PROTOBUF_DEP) $(PROTOC_PLUGINS) $(E) "[GRPC] Generating gRPC's protobuf service CC file from $<" $(Q) mkdir -p `dirname $@` $(Q) $(PROTOC) --grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(BINDIR)/$(CONFIG)/grpc_cpp_plugin $< @@ -3532,9 +3532,9 @@ endif LIBGRPC++_BENCHMARK_CONFIG_SRC = \ $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc \ - $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc \ + $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc \ + test/cpp/qps/perf_db_client.cc \ test/cpp/qps/report.cc \ - test/cpp/qps/user_data_client.cc \ test/cpp/util/benchmark_config.cc \ @@ -3579,9 +3579,9 @@ ifneq ($(NO_DEPS),true) -include $(LIBGRPC++_BENCHMARK_CONFIG_OBJS:.o=.dep) endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/user_data_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/perf_db_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc LIBGRPC++_TEST_CONFIG_SRC = \ @@ -4094,15 +4094,15 @@ $(OBJDIR)/$(CONFIG)/examples/pubsub/subscriber.o: $(GENDIR)/examples/pubsub/labe LIBQPS_SRC = \ $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc \ - $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc \ + $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc \ test/cpp/qps/client_async.cc \ test/cpp/qps/client_sync.cc \ test/cpp/qps/driver.cc \ + test/cpp/qps/perf_db_client.cc \ test/cpp/qps/qps_worker.cc \ test/cpp/qps/server_async.cc \ test/cpp/qps/server_sync.cc \ test/cpp/qps/timer.cc \ - test/cpp/qps/user_data_client.cc \ LIBQPS_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBQPS_SRC)))) @@ -4146,14 +4146,14 @@ ifneq ($(NO_DEPS),true) -include $(LIBQPS_OBJS:.o=.dep) endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/timer.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/user_data_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/user_data.pb.cc $(GENDIR)/test/cpp/qps/user_data.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/perf_db_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/timer.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc LIBGRPC_CSHARP_EXT_SRC = \ diff --git a/build.json b/build.json index 9342aea0355..3c8dd756cac 100644 --- a/build.json +++ b/build.json @@ -541,9 +541,9 @@ "language": "c++", "src": [ "test/cpp/qps/qpstest.proto", - "test/cpp/qps/user_data.proto", + "test/cpp/qps/perf_db.proto", + "test/cpp/qps/perf_db_client.cc", "test/cpp/qps/report.cc", - "test/cpp/qps/user_data_client.cc", "test/cpp/util/benchmark_config.cc" ] }, @@ -714,15 +714,15 @@ ], "src": [ "test/cpp/qps/qpstest.proto", - "test/cpp/qps/user_data.proto", + "test/cpp/qps/perf_db.proto", "test/cpp/qps/client_async.cc", "test/cpp/qps/client_sync.cc", "test/cpp/qps/driver.cc", + "test/cpp/qps/perf_db_client.cc", "test/cpp/qps/qps_worker.cc", "test/cpp/qps/server_async.cc", "test/cpp/qps/server_sync.cc", - "test/cpp/qps/timer.cc", - "test/cpp/qps/user_data_client.cc" + "test/cpp/qps/timer.cc" ], "deps": [ "grpc_test_util", diff --git a/test/cpp/qps/user_data.proto b/test/cpp/qps/perf_db.proto similarity index 99% rename from test/cpp/qps/user_data.proto rename to test/cpp/qps/perf_db.proto index 947ccce60b6..f4f174937c0 100644 --- a/test/cpp/qps/user_data.proto +++ b/test/cpp/qps/perf_db.proto @@ -33,7 +33,7 @@ import "test/cpp/qps/qpstest.proto"; package grpc.testing; -service UserDataTransfer { +service PerfDbTransfer { // Sends client info rpc RecordSingleClientData (SingleUserRecordRequest) returns (SingleUserRecordReply) {} diff --git a/test/cpp/qps/user_data_client.cc b/test/cpp/qps/perf_db_client.cc similarity index 86% rename from test/cpp/qps/user_data_client.cc rename to test/cpp/qps/perf_db_client.cc index a8ced7559d7..6a1fe7e26e7 100644 --- a/test/cpp/qps/user_data_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -31,30 +31,30 @@ * */ -#include "user_data_client.h" +#include "perf_db_client.h" namespace grpc { namespace testing { //sets the client and server config information -void UserDataClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) { +void PerfDbClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) const { clientConfig_ = clientConfig; serverConfig_ = serverConfig; } //sets the QPS -void UserDataClient::setQPS(double QPS) { +void PerfDbClient::setQPS(double QPS) const { QPS_ = QPS; } //sets the QPS per core -void UserDataClient::setQPSPerCore(double QPSPerCore) { +void PerfDbClient::setQPSPerCore(double QPSPerCore) const { QPSPerCore_ = QPSPerCore; } //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency -void UserDataClient::setLatencies(double percentileLatency50, double percentileLatency90, - double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) { +void PerfDbClient::setLatencies(double percentileLatency50, double percentileLatency90, + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) const { percentileLatency50_ = percentileLatency50; percentileLatency90_ = percentileLatency90; percentileLatency95_ = percentileLatency95; @@ -63,8 +63,8 @@ void UserDataClient::setLatencies(double percentileLatency50, double percentileL } //sets the server and client, user and system times -void UserDataClient::setTimes(double serverSystemTime, double serverUserTime, - double clientSystemTime, double clientUserTime) { +void PerfDbClient::setTimes(double serverSystemTime, double serverUserTime, + double clientSystemTime, double clientUserTime) const { serverSystemTime_ = serverSystemTime; serverUserTime_ = serverUserTime; clientSystemTime_ = clientSystemTime; @@ -72,7 +72,7 @@ void UserDataClient::setTimes(double serverSystemTime, double serverUserTime, } //sends the data to the performancew database server -int UserDataClient::sendData(std::string access_token, std::string test_name, std::string sys_info) { +int PerfDbClient::sendData(std::string access_token, std::string test_name, std::string sys_info) const { //Data record request object SingleUserRecordRequest singleUserRecordRequest; @@ -104,7 +104,7 @@ int UserDataClient::sendData(std::string access_token, std::string test_name, st ClientContext context; Status status = stub_->RecordSingleClientData(&context, singleUserRecordRequest, &singleUserRecordReply); - if (status.IsOk()) { + if (status.ok()) { return 1; //data sent to database successfully } else { return -1; //error in data sending diff --git a/test/cpp/qps/user_data_client.h b/test/cpp/qps/perf_db_client.h similarity index 71% rename from test/cpp/qps/user_data_client.h rename to test/cpp/qps/perf_db_client.h index c2e07ef5cd9..4a63dbf3d28 100644 --- a/test/cpp/qps/user_data_client.h +++ b/test/cpp/qps/perf_db_client.h @@ -43,55 +43,56 @@ #include #include #include -#include "test/cpp/qps/user_data.grpc.pb.h" +#include "test/cpp/qps/perf_db.grpc.pb.h" namespace grpc{ namespace testing { //Manages data sending to performance database server -class UserDataClient { +class PerfDbClient { public: - UserDataClient(std::shared_ptr channel) - : stub_(UserDataTransfer::NewStub(channel)) {} + PerfDbClient() {} - ~UserDataClient() {} + void init(std::shared_ptr channel) { stub_ = PerfDbTransfer::NewStub(channel); } + + ~PerfDbClient() {} //sets the client and server config information - void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig); + void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) const; //sets the QPS - void setQPS(double QPS); + void setQPS(double QPS) const; //sets the QPS per core - void setQPSPerCore(double QPSPerCore); + void setQPSPerCore(double QPSPerCore) const; //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency void setLatencies(double percentileLatency50, double percentileLatency90, - double percentileLatency95, double percentileLatency99, double percentileLatency99Point9); + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) const; //sets the server and client, user and system times void setTimes(double serverSystemTime, double serverUserTime, - double clientSystemTime, double clientUserTime); + double clientSystemTime, double clientUserTime) const; //sends the data to the performancew database server - int sendData(std::string access_token, std::string test_name, std::string sys_info); + int sendData(std::string access_token, std::string test_name, std::string sys_info) const; private: - std::unique_ptr stub_; - ClientConfig clientConfig_; - ServerConfig serverConfig_; - double QPS_ = DBL_MIN; - double QPSPerCore_ = DBL_MIN; - double percentileLatency50_ = DBL_MIN; - double percentileLatency90_ = DBL_MIN; - double percentileLatency95_ = DBL_MIN; - double percentileLatency99_ = DBL_MIN; - double percentileLatency99Point9_ = DBL_MIN; - double serverSystemTime_ = DBL_MIN; - double serverUserTime_ = DBL_MIN; - double clientSystemTime_ = DBL_MIN; - double clientUserTime_ = DBL_MIN; + std::unique_ptr stub_; + mutable ClientConfig clientConfig_; + mutable ServerConfig serverConfig_; + mutable double QPS_ = DBL_MIN; + mutable double QPSPerCore_ = DBL_MIN; + mutable double percentileLatency50_ = DBL_MIN; + mutable double percentileLatency90_ = DBL_MIN; + mutable double percentileLatency95_ = DBL_MIN; + mutable double percentileLatency99_ = DBL_MIN; + mutable double percentileLatency99Point9_ = DBL_MIN; + mutable double serverSystemTime_ = DBL_MIN; + mutable double serverUserTime_ = DBL_MIN; + mutable double clientSystemTime_ = DBL_MIN; + mutable double clientUserTime_ = DBL_MIN; }; } //namespace testing diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index c524327ff03..3d71f76fd22 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -35,7 +35,6 @@ #include #include "test/cpp/qps/stats.h" -#include "user_data_client.h" namespace grpc { namespace testing { @@ -119,41 +118,41 @@ void GprLogReporter::ReportTimes(const ScenarioResult& result) const { [](ResourceUsage u) { return u.wall_time; })); } -UserDataClient userDataClient(grpc::CreateChannel("localhost:50052", grpc::InsecureCredentials(), - ChannelArguments())); +// perfDbClient perfDbClient(grpc::CreateChannel("localhost:50052", grpc::InsecureCredentials(), +// ChannelArguments())); //Performance database reporter implementation. -void UserDatabaseReporter::ReportQPS(const ScenarioResult& result) const { +void PerfDbReporter::ReportQPS(const ScenarioResult& result) const { auto qps = result.latencies.Count() / average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); - userDataClient.setQPS(qps); - userDataClient.setConfigs(result.client_config, result.server_config); + perfDbClient.setQPS(qps); + perfDbClient.setConfigs(result.client_config, result.server_config); } -void UserDatabaseReporter::ReportQPSPerCore(const ScenarioResult& result) const { +void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) const { auto qps = result.latencies.Count() / average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); auto qpsPerCore = qps / result.server_config.threads(); - userDataClient.setQPS(qps); - userDataClient.setQPSPerCore(qpsPerCore); - userDataClient.setConfigs(result.client_config, result.server_config); + perfDbClient.setQPS(qps); + perfDbClient.setQPSPerCore(qpsPerCore); + perfDbClient.setConfigs(result.client_config, result.server_config); } -void UserDatabaseReporter::ReportLatency(const ScenarioResult& result) const { - userDataClient.setLatencies(result.latencies.Percentile(50) / 1000, +void PerfDbReporter::ReportLatency(const ScenarioResult& result) const { + perfDbClient.setLatencies(result.latencies.Percentile(50) / 1000, result.latencies.Percentile(90) / 1000, result.latencies.Percentile(95) / 1000, result.latencies.Percentile(99) / 1000, result.latencies.Percentile(99.9) / 1000); - userDataClient.setConfigs(result.client_config, result.server_config); + perfDbClient.setConfigs(result.client_config, result.server_config); } -void UserDatabaseReporter::ReportTimes(const ScenarioResult& result) const { +void PerfDbReporter::ReportTimes(const ScenarioResult& result) const { double serverSystemTime = 100.0 * sum(result.server_resources, [](ResourceUsage u) { return u.system_time; }) / sum(result.server_resources, @@ -171,22 +170,22 @@ void UserDatabaseReporter::ReportTimes(const ScenarioResult& result) const { sum(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); - userDataClient.setTimes(serverSystemTime, serverUserTime, + perfDbClient.setTimes(serverSystemTime, serverUserTime, clientSystemTime, clientUserTime); - userDataClient.setConfigs(result.client_config, result.server_config); + perfDbClient.setConfigs(result.client_config, result.server_config); } -void UserDatabaseReporter::SendData() const { +void PerfDbReporter::SendData() const { //send data to performance database - int userDataState = userDataClient.sendData(access_token_, test_name_, sys_info_); + int dataState = perfDbClient.sendData(access_token_, test_name_, sys_info_); //check state of data sending - switch(userDataState) { + switch(dataState) { case 1: - gpr_log(GPR_INFO, "Data sent to user database successfully"); + gpr_log(GPR_INFO, "Data sent to performance database successfully"); break; case -1: - gpr_log(GPR_INFO, "Data could not be sent to user database"); + gpr_log(GPR_INFO, "Data could not be sent to performance database"); break; } } diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 432ed5ddf2b..afc79ff75fc 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -41,6 +41,7 @@ #include "test/cpp/qps/driver.h" #include "test/cpp/qps/qpstest.grpc.pb.h" +#include "perf_db_client.h" namespace grpc { namespace testing { @@ -104,13 +105,16 @@ class GprLogReporter : public Reporter { }; /** Reporter for client leaderboard. */ -class UserDatabaseReporter : public Reporter { +class PerfDbReporter : public Reporter { public: - UserDatabaseReporter(const string& name, const string& access_token, const string& test_name, const string& sys_info) - : Reporter(name), access_token_(access_token), test_name_(test_name), sys_info_(sys_info) {} - ~UserDatabaseReporter() { SendData(); }; + PerfDbReporter(const string& name, const string& access_token, const string& test_name, const string& sys_info, const string& server_address) + : Reporter(name), access_token_(access_token), test_name_(test_name), sys_info_(sys_info) { + perfDbClient.init(grpc::CreateChannel(server_address, grpc::InsecureCredentials(), ChannelArguments())); + } + ~PerfDbReporter() { SendData(); }; private: + PerfDbClient perfDbClient; std::string access_token_; std::string test_name_; std::string sys_info_; diff --git a/test/cpp/qps/run_perf_db_test.py b/test/cpp/qps/run_perf_db_test.py index 7f840fa723d..34007aa97c9 100755 --- a/test/cpp/qps/run_perf_db_test.py +++ b/test/cpp/qps/run_perf_db_test.py @@ -257,6 +257,8 @@ def main(): except Exception, e: print 'Error in authentication' + serverAddress = 'sidrakesh.mtv.google.corp.com:50052' + try: testPath = findTestPath(test) # Get path to test testName = testPath.split('/')[-1] # Get test name @@ -265,7 +267,7 @@ def main(): print '\nBeginning test:\n' # Run the test - subprocess.call([testPath, '--report_metrics_db=true', '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]')]) + subprocess.call([testPath, '--report_metrics_db=true', '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]'), '--server_address='+serverAddress]) except OSError: print 'Could not execute the test, please check test name' diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc index 250bb7ac8e3..f8cfabd0d68 100644 --- a/test/cpp/util/benchmark_config.cc +++ b/test/cpp/util/benchmark_config.cc @@ -45,6 +45,8 @@ DEFINE_string(test_name, "", "Name of the test being executed"); DEFINE_string(sys_info, "", "System information"); +DEFINE_string(server_address, "localhost:50052", "Address of the performance database server"); + // In some distros, gflags is in the namespace google, and in some others, // in gflags. This hack is enabling us to find both. namespace google {} @@ -67,7 +69,7 @@ static std::shared_ptr InitBenchmarkReporters() { } if(FLAGS_report_metrics_db) { composite_reporter->add( - std::unique_ptr(new UserDatabaseReporter("UserDataReporter", FLAGS_access_token, FLAGS_test_name, FLAGS_sys_info))); + std::unique_ptr(new PerfDbReporter("PerfDbReporter", FLAGS_access_token, FLAGS_test_name, FLAGS_sys_info, FLAGS_server_address))); } return std::shared_ptr(composite_reporter); From 301a77f819234f57cc5713fe748860122c1b44c2 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 18 Jun 2015 17:02:31 -0700 Subject: [PATCH 16/63] Modified travis config for ruby. In order to output logs. --- tools/run_tests/build_ruby.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/build_ruby.sh b/tools/run_tests/build_ruby.sh index de96413bc12..da6121d99ca 100755 --- a/tools/run_tests/build_ruby.sh +++ b/tools/run_tests/build_ruby.sh @@ -37,4 +37,4 @@ export CONFIG=${CONFIG:-opt} cd $(dirname $0)/../../src/ruby bundle install -rake compile:grpc +rake compile:grpc || cat tmp/*/*/*/mkmf.log From 12c65b1a7e3d6707b703d6294144c63c9dac122d Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Fri, 19 Jun 2015 11:16:12 -0700 Subject: [PATCH 17/63] slightly updated tool script --- test/cpp/qps/run_perf_db_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/run_perf_db_test.py b/test/cpp/qps/run_perf_db_test.py index 34007aa97c9..634555546a3 100755 --- a/test/cpp/qps/run_perf_db_test.py +++ b/test/cpp/qps/run_perf_db_test.py @@ -230,7 +230,7 @@ def getSysInfo(): tcp_rr_rate = str(os.popen('netperf -t TCP_RR -v 0').readlines()[1]) print 'Network info obtained' - nicInfo.append('TCP RR Transmission Rate per sec: ' + tcp_rr_rate + '\n') + nicInfo.append('TCP RR transmission rate per sec: ' + tcp_rr_rate + '\n') sysInfo = sysInfo + nicInfo return sysInfo @@ -257,13 +257,14 @@ def main(): except Exception, e: print 'Error in authentication' + # Address of the performance database server serverAddress = 'sidrakesh.mtv.google.corp.com:50052' try: testPath = findTestPath(test) # Get path to test testName = testPath.split('/')[-1] # Get test name - sysInfo = getSysInfo() + sysInfo = getSysInfo() # get the system information print '\nBeginning test:\n' # Run the test From 31014d26aa2757de26e9f395bab84220a593070e Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Fri, 19 Jun 2015 15:24:19 -0700 Subject: [PATCH 18/63] flags handled using python_gflags --- test/cpp/qps/run_perf_db_test.py | 73 ++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/test/cpp/qps/run_perf_db_test.py b/test/cpp/qps/run_perf_db_test.py index 634555546a3..fd0d397a66f 100755 --- a/test/cpp/qps/run_perf_db_test.py +++ b/test/cpp/qps/run_perf_db_test.py @@ -39,15 +39,23 @@ import json import time import subprocess import fnmatch +import gflags CLIENT_ID = '1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' CLIENT_SECRET = '_HGHXg4DAA59r4w4x8p6ARzD' GRANT_TYPE = 'http://oauth.net/grant_type/device/1.0' -ACCESS_TOKENS_DIR = '/usr/local/auth_access_tokens' +DEFAULT_ACCESS_TOKENS_DIR = '/usr/local/auth_access_tokens' AUTH_TOKEN_LINK = 'https://www.googleapis.com/oauth2/v3/token' GOOGLE_ACCOUNTS_LINK = 'https://accounts.google.com/o/oauth2/device/code' USER_INFO_LINK = 'https://www.googleapis.com/oauth2/v1/userinfo' +FLAGS = gflags.FLAGS + +gflags.DEFINE_string('test', None, 'Name of the test') +gflags.DEFINE_string('email', None, 'gmail id') +gflags.DEFINE_string('server_address', 'localhost:50052', 'Address of the performance database server') +gflags.DEFINE_string('tokens_dir', DEFAULT_ACCESS_TOKENS_DIR, 'Location of the access tokens') + # Fetches JSON reply object, given a url and parameters def fetchJSON(url, paramDict): if len(paramDict) == 0: @@ -136,7 +144,7 @@ def refreshAccessToken(refreshToken, userTokFile): # return fresh access token return data['access_token'] -def reauthenticate(): +def reauthenticate(tokensDir): # Request parameters paramDict = {'client_id':CLIENT_ID, 'scope':'email profile'} JSONBody = fetchJSON(GOOGLE_ACCOUNTS_LINK, paramDict) @@ -158,7 +166,7 @@ def reauthenticate(): time.sleep(data['interval']) # File to write tokens - newUserTokFile = ACCESS_TOKENS_DIR + '/' + getUserId(authData['access_token']) + newUserTokFile = tokensDir + '/' + getUserId(authData['access_token']) # Write tokens to file with open(newUserTokFile, "w") as data_file: @@ -168,12 +176,12 @@ def reauthenticate(): return authData['access_token'] # Fetch a working access token given user entered email id; authntication may be required -def getAccessToken(email): +def getAccessToken(email, tokensDir): # Get unique user id from email address userId = getUserIdFromEmail(email) # Token file - userTokFile = ACCESS_TOKENS_DIR + '/' + userId + userTokFile = tokensDir + '/' + userId accessToken = '' @@ -182,7 +190,7 @@ def getAccessToken(email): accessToken = useAccessToken(userTokFile) else: # User authentication required - accessToken = reauthenticate() + accessToken = reauthenticate(tokensDir) return accessToken @@ -235,42 +243,51 @@ def getSysInfo(): return sysInfo -def main(): - # If tokens directory does not exist, creates it - if not os.path.exists(ACCESS_TOKENS_DIR): - subprocess.call(['sudo', 'mkdir', ACCESS_TOKENS_DIR]) - subprocess.call(['sudo', 'chmod', '777', ACCESS_TOKENS_DIR]) +def main(argv): + try: + argv = FLAGS(argv) + except Exception, e: + print '%s\\nUsage: %s ARGS\\n%s' % (e, sys.argv[0], FLAGS) + sys.exit(1) - if len(sys.argv) > 1: - test = sys.argv[1] - else: - test = raw_input('Enter the test path/name: ') + tokensDir = FLAGS.tokens_dir - if len(sys.argv) > 2: - email = sys.argv[2] - else: - email = raw_input('Enter your e-mail id: ') + # If tokens directory does not exist, creates it + if not os.path.exists(tokensDir): + os.makedirs(tokensDir) try: # Fetch working access token - accessToken = getAccessToken(email) - except Exception, e: - print 'Error in authentication' + accessToken = getAccessToken(FLAGS.email, tokensDir) + except AttributeError: + print '\nError: Please provide email address as an argument\n' + sys.exit(1) + except: + print '\nError in authentication\n' + sys.exit(1) # Address of the performance database server - serverAddress = 'sidrakesh.mtv.google.corp.com:50052' + serverAddress = FLAGS.server_address + # Get path to test try: - testPath = findTestPath(test) # Get path to test - testName = testPath.split('/')[-1] # Get test name + testPath = findTestPath(FLAGS.test) + except TypeError: + print '\nError: Please provide test name/path as argument\n' + sys.exit(1) - sysInfo = getSysInfo() # get the system information + # Get name of the test + testName = testPath.split('/')[-1] + # Get the system information + sysInfo = getSysInfo() + + try: print '\nBeginning test:\n' # Run the test subprocess.call([testPath, '--report_metrics_db=true', '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]'), '--server_address='+serverAddress]) except OSError: - print 'Could not execute the test, please check test name' + print 'Could not execute the test' if __name__ == "__main__": - main() \ No newline at end of file + main(sys.argv) \ No newline at end of file From 64967d4faccda02b2cd2cfb6c7a7fedab1930ebf Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Fri, 19 Jun 2015 16:58:26 -0700 Subject: [PATCH 19/63] removed dependency on python_gflags, using argparse instead --- test/cpp/qps/run_perf_db_test.py | 42 +++++++++++++++----------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/test/cpp/qps/run_perf_db_test.py b/test/cpp/qps/run_perf_db_test.py index fd0d397a66f..a8e69ea1f02 100755 --- a/test/cpp/qps/run_perf_db_test.py +++ b/test/cpp/qps/run_perf_db_test.py @@ -39,22 +39,20 @@ import json import time import subprocess import fnmatch -import gflags +import argparse CLIENT_ID = '1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' CLIENT_SECRET = '_HGHXg4DAA59r4w4x8p6ARzD' GRANT_TYPE = 'http://oauth.net/grant_type/device/1.0' -DEFAULT_ACCESS_TOKENS_DIR = '/usr/local/auth_access_tokens' AUTH_TOKEN_LINK = 'https://www.googleapis.com/oauth2/v3/token' GOOGLE_ACCOUNTS_LINK = 'https://accounts.google.com/o/oauth2/device/code' USER_INFO_LINK = 'https://www.googleapis.com/oauth2/v1/userinfo' -FLAGS = gflags.FLAGS - -gflags.DEFINE_string('test', None, 'Name of the test') -gflags.DEFINE_string('email', None, 'gmail id') -gflags.DEFINE_string('server_address', 'localhost:50052', 'Address of the performance database server') -gflags.DEFINE_string('tokens_dir', DEFAULT_ACCESS_TOKENS_DIR, 'Location of the access tokens') +parser = argparse.ArgumentParser(description='Report metrics to performance database') +parser.add_argument('--test', type=str, help='Name of the test to be executed') +parser.add_argument('--email', type=str, help='Gmail address of the user') +parser.add_argument('--server_address', type=str, default='localhost:50052', help='Address of the performance database server') +parser.add_argument('--tokens_dir', type=str, default=os.path.expanduser('~')+'/.grpc/access_tokens', help='Path to the access tokens directory') # Fetches JSON reply object, given a url and parameters def fetchJSON(url, paramDict): @@ -145,6 +143,11 @@ def refreshAccessToken(refreshToken, userTokFile): return data['access_token'] def reauthenticate(tokensDir): + # Create folder if not created already + if not os.path.exists(tokensDir): + os.makedirs(tokensDir) + os.chmod(tokensDir, 0700) + # Request parameters paramDict = {'client_id':CLIENT_ID, 'scope':'email profile'} JSONBody = fetchJSON(GOOGLE_ACCOUNTS_LINK, paramDict) @@ -170,6 +173,7 @@ def reauthenticate(tokensDir): # Write tokens to file with open(newUserTokFile, "w") as data_file: + os.chmod(newUserTokFile, 0600) json.dump(authData, data_file) # return working access token @@ -244,34 +248,26 @@ def getSysInfo(): return sysInfo def main(argv): - try: - argv = FLAGS(argv) - except Exception, e: - print '%s\\nUsage: %s ARGS\\n%s' % (e, sys.argv[0], FLAGS) - sys.exit(1) + args = parser.parse_args() - tokensDir = FLAGS.tokens_dir - - # If tokens directory does not exist, creates it - if not os.path.exists(tokensDir): - os.makedirs(tokensDir) + tokensDir = args.tokens_dir try: # Fetch working access token - accessToken = getAccessToken(FLAGS.email, tokensDir) + accessToken = getAccessToken(args.email, tokensDir) except AttributeError: print '\nError: Please provide email address as an argument\n' sys.exit(1) - except: - print '\nError in authentication\n' + except Exception, e: + print e, ' \nError in authentication\n' sys.exit(1) # Address of the performance database server - serverAddress = FLAGS.server_address + serverAddress = args.server_address # Get path to test try: - testPath = findTestPath(FLAGS.test) + testPath = findTestPath(args.test) except TypeError: print '\nError: Please provide test name/path as argument\n' sys.exit(1) From c3cd4780df01e55a145c571074c5f31b13d9d864 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Fri, 19 Jun 2015 17:05:19 -0700 Subject: [PATCH 20/63] minor commented code removal --- test/cpp/qps/report.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 3d71f76fd22..b8ad4cf332a 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -118,10 +118,6 @@ void GprLogReporter::ReportTimes(const ScenarioResult& result) const { [](ResourceUsage u) { return u.wall_time; })); } -// perfDbClient perfDbClient(grpc::CreateChannel("localhost:50052", grpc::InsecureCredentials(), -// ChannelArguments())); - -//Performance database reporter implementation. void PerfDbReporter::ReportQPS(const ScenarioResult& result) const { auto qps = result.latencies.Count() / average(result.client_resources, From 373ad4d0d0ab1d9f970fc9a8d51526db2b48952b Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 22 Jun 2015 11:36:17 -0700 Subject: [PATCH 21/63] edited report --- test/cpp/qps/report.cc | 20 ++++++++++---------- test/cpp/qps/report.h | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index b8ad4cf332a..9bb15ea2111 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -123,8 +123,8 @@ void PerfDbReporter::ReportQPS(const ScenarioResult& result) const { average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); - perfDbClient.setQPS(qps); - perfDbClient.setConfigs(result.client_config, result.server_config); + perfDbClient_.setQPS(qps); + perfDbClient_.setConfigs(result.client_config, result.server_config); } void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) const { @@ -134,18 +134,18 @@ void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) const { auto qpsPerCore = qps / result.server_config.threads(); - perfDbClient.setQPS(qps); - perfDbClient.setQPSPerCore(qpsPerCore); - perfDbClient.setConfigs(result.client_config, result.server_config); + perfDbClient_.setQPS(qps); + perfDbClient_.setQPSPerCore(qpsPerCore); + perfDbClient_.setConfigs(result.client_config, result.server_config); } void PerfDbReporter::ReportLatency(const ScenarioResult& result) const { - perfDbClient.setLatencies(result.latencies.Percentile(50) / 1000, + perfDbClient_.setLatencies(result.latencies.Percentile(50) / 1000, result.latencies.Percentile(90) / 1000, result.latencies.Percentile(95) / 1000, result.latencies.Percentile(99) / 1000, result.latencies.Percentile(99.9) / 1000); - perfDbClient.setConfigs(result.client_config, result.server_config); + perfDbClient_.setConfigs(result.client_config, result.server_config); } void PerfDbReporter::ReportTimes(const ScenarioResult& result) const { @@ -166,14 +166,14 @@ void PerfDbReporter::ReportTimes(const ScenarioResult& result) const { sum(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); - perfDbClient.setTimes(serverSystemTime, serverUserTime, + perfDbClient_.setTimes(serverSystemTime, serverUserTime, clientSystemTime, clientUserTime); - perfDbClient.setConfigs(result.client_config, result.server_config); + perfDbClient_.setConfigs(result.client_config, result.server_config); } void PerfDbReporter::SendData() const { //send data to performance database - int dataState = perfDbClient.sendData(access_token_, test_name_, sys_info_); + int dataState = perfDbClient_.sendData(access_token_, test_name_, sys_info_); //check state of data sending switch(dataState) { diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index afc79ff75fc..1ea33755d45 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -41,7 +41,7 @@ #include "test/cpp/qps/driver.h" #include "test/cpp/qps/qpstest.grpc.pb.h" -#include "perf_db_client.h" +#include "test/cpp/qps/perf_db_client.h" namespace grpc { namespace testing { @@ -104,7 +104,7 @@ class GprLogReporter : public Reporter { void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; }; -/** Reporter for client leaderboard. */ +/** Reporter for performance database tool */ class PerfDbReporter : public Reporter { public: PerfDbReporter(const string& name, const string& access_token, const string& test_name, const string& sys_info, const string& server_address) @@ -114,7 +114,7 @@ class PerfDbReporter : public Reporter { ~PerfDbReporter() { SendData(); }; private: - PerfDbClient perfDbClient; + PerfDbClient perfDbClient_; std::string access_token_; std::string test_name_; std::string sys_info_; From 43645466681022f31bf2305b01bd7fab25369695 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 22 Jun 2015 12:21:57 -0700 Subject: [PATCH 22/63] Applied required modifications --- test/cpp/qps/perf_db.proto | 78 ++++----- test/cpp/qps/perf_db_client.cc | 14 +- test/cpp/qps/perf_db_client.h | 38 ++-- test/cpp/qps/report.cc | 10 +- test/cpp/qps/report.h | 12 +- test/cpp/qps/run_perf_db_test.py | 289 ------------------------------- 6 files changed, 76 insertions(+), 365 deletions(-) delete mode 100755 test/cpp/qps/run_perf_db_test.py diff --git a/test/cpp/qps/perf_db.proto b/test/cpp/qps/perf_db.proto index f4f174937c0..50070fda5b6 100644 --- a/test/cpp/qps/perf_db.proto +++ b/test/cpp/qps/perf_db.proto @@ -27,7 +27,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -syntax = "proto2"; +syntax = "proto3"; import "test/cpp/qps/qpstest.proto"; @@ -44,58 +44,58 @@ service PerfDbTransfer { //Metrics to be stored message Metrics { - optional double qps = 1; - optional double qps_per_core = 2; - optional double perc_lat_50 = 3; - optional double perc_lat_90 = 4; - optional double perc_lat_95 = 5; - optional double perc_lat_99 = 6; - optional double perc_lat_99_point_9 = 7; - optional double server_system_time = 8; - optional double server_user_time = 9; - optional double client_system_time = 10; - optional double client_user_time = 11; + double qps = 1; + double qps_per_core = 2; + double perc_lat_50 = 3; + double perc_lat_90 = 4; + double perc_lat_95 = 5; + double perc_lat_99 = 6; + double perc_lat_99_point_9 = 7; + double server_system_time = 8; + double server_user_time = 9; + double client_system_time = 10; + double client_user_time = 11; } //Timestamped details message DataDetails { - optional string timestamp = 1; - optional string test_name = 2; - optional string sys_info = 3; - optional Metrics metrics = 4; - optional ClientConfig client_config = 5; - optional ServerConfig server_config = 6; + string timestamp = 1; + string test_name = 2; + string sys_info = 3; + Metrics metrics = 4; + ClientConfig client_config = 5; + ServerConfig server_config = 6; } //User details message UserDetails { - optional string id = 1; - optional string email = 2; - optional bool verified_email = 3; - optional string name = 4; - optional string given_name = 5; - optional string family_name = 6; - optional string link = 7; - optional string picture = 8; - optional string gender = 9; - optional string locale = 10; - optional string hd = 11; + string id = 1; + string email = 2; + bool verified_email = 3; + string name = 4; + string given_name = 5; + string family_name = 6; + string link = 7; + string picture = 8; + string gender = 9; + string locale = 10; + string hd = 11; } //Stored to database message SingleUserDetails { repeated DataDetails data_details = 1; - optional UserDetails user_details = 2; + UserDetails user_details = 2; } //Request for storing a single user's data message SingleUserRecordRequest { - optional string access_token = 1; - optional string test_name = 2; - optional string sys_info = 3; - optional Metrics metrics = 4; - optional ClientConfig client_config = 5; - optional ServerConfig server_config = 6; + string access_token = 1; + string test_name = 2; + string sys_info = 3; + Metrics metrics = 4; + ClientConfig client_config = 5; + ServerConfig server_config = 6; } //Reply to request for storing single user's data @@ -104,12 +104,12 @@ message SingleUserRecordReply { //Request for retrieving single user's data message SingleUserRetrieveRequest { - optional string user_id = 1; + string user_id = 1; } //Reply for request to retrieve single user's data message SingleUserRetrieveReply { - optional SingleUserDetails details = 1; + SingleUserDetails details = 1; } //Request for retrieving all users' data @@ -119,4 +119,4 @@ message AllUsersRetrieveReply { //Reply to request for retrieving all users' data message AllUsersRetrieveRequest { -} \ No newline at end of file +} diff --git a/test/cpp/qps/perf_db_client.cc b/test/cpp/qps/perf_db_client.cc index 6a1fe7e26e7..5b929087669 100644 --- a/test/cpp/qps/perf_db_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -31,30 +31,30 @@ * */ -#include "perf_db_client.h" +#include "test/cpp/qps/perf_db_client.h" namespace grpc { namespace testing { //sets the client and server config information -void PerfDbClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) const { +void PerfDbClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) { clientConfig_ = clientConfig; serverConfig_ = serverConfig; } //sets the QPS -void PerfDbClient::setQPS(double QPS) const { +void PerfDbClient::setQPS(double QPS) { QPS_ = QPS; } //sets the QPS per core -void PerfDbClient::setQPSPerCore(double QPSPerCore) const { +void PerfDbClient::setQPSPerCore(double QPSPerCore) { QPSPerCore_ = QPSPerCore; } //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency void PerfDbClient::setLatencies(double percentileLatency50, double percentileLatency90, - double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) const { + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) { percentileLatency50_ = percentileLatency50; percentileLatency90_ = percentileLatency90; percentileLatency95_ = percentileLatency95; @@ -64,7 +64,7 @@ void PerfDbClient::setLatencies(double percentileLatency50, double percentileLat //sets the server and client, user and system times void PerfDbClient::setTimes(double serverSystemTime, double serverUserTime, - double clientSystemTime, double clientUserTime) const { + double clientSystemTime, double clientUserTime) { serverSystemTime_ = serverSystemTime; serverUserTime_ = serverUserTime; clientSystemTime_ = clientSystemTime; @@ -72,7 +72,7 @@ void PerfDbClient::setTimes(double serverSystemTime, double serverUserTime, } //sends the data to the performancew database server -int PerfDbClient::sendData(std::string access_token, std::string test_name, std::string sys_info) const { +int PerfDbClient::sendData(std::string access_token, std::string test_name, std::string sys_info) { //Data record request object SingleUserRecordRequest singleUserRecordRequest; diff --git a/test/cpp/qps/perf_db_client.h b/test/cpp/qps/perf_db_client.h index 4a63dbf3d28..058cd6a2070 100644 --- a/test/cpp/qps/perf_db_client.h +++ b/test/cpp/qps/perf_db_client.h @@ -59,40 +59,40 @@ public: ~PerfDbClient() {} //sets the client and server config information - void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) const; + void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig); //sets the QPS - void setQPS(double QPS) const; + void setQPS(double QPS); //sets the QPS per core - void setQPSPerCore(double QPSPerCore) const; + void setQPSPerCore(double QPSPerCore); //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency void setLatencies(double percentileLatency50, double percentileLatency90, - double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) const; + double percentileLatency95, double percentileLatency99, double percentileLatency99Point9); //sets the server and client, user and system times void setTimes(double serverSystemTime, double serverUserTime, - double clientSystemTime, double clientUserTime) const; + double clientSystemTime, double clientUserTime); //sends the data to the performancew database server - int sendData(std::string access_token, std::string test_name, std::string sys_info) const; + int sendData(std::string access_token, std::string test_name, std::string sys_info); private: std::unique_ptr stub_; - mutable ClientConfig clientConfig_; - mutable ServerConfig serverConfig_; - mutable double QPS_ = DBL_MIN; - mutable double QPSPerCore_ = DBL_MIN; - mutable double percentileLatency50_ = DBL_MIN; - mutable double percentileLatency90_ = DBL_MIN; - mutable double percentileLatency95_ = DBL_MIN; - mutable double percentileLatency99_ = DBL_MIN; - mutable double percentileLatency99Point9_ = DBL_MIN; - mutable double serverSystemTime_ = DBL_MIN; - mutable double serverUserTime_ = DBL_MIN; - mutable double clientSystemTime_ = DBL_MIN; - mutable double clientUserTime_ = DBL_MIN; + ClientConfig clientConfig_; + ServerConfig serverConfig_; + double QPS_ = DBL_MIN; + double QPSPerCore_ = DBL_MIN; + double percentileLatency50_ = DBL_MIN; + double percentileLatency90_ = DBL_MIN; + double percentileLatency95_ = DBL_MIN; + double percentileLatency99_ = DBL_MIN; + double percentileLatency99Point9_ = DBL_MIN; + double serverSystemTime_ = DBL_MIN; + double serverUserTime_ = DBL_MIN; + double clientSystemTime_ = DBL_MIN; + double clientUserTime_ = DBL_MIN; }; } //namespace testing diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 47d602a4c2e..59c0da31fb9 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -118,7 +118,7 @@ void GprLogReporter::ReportTimes(const ScenarioResult& result) { [](ResourceUsage u) { return u.wall_time; })); } -void PerfDbReporter::ReportQPS(const ScenarioResult& result) const { +void PerfDbReporter::ReportQPS(const ScenarioResult& result) { auto qps = result.latencies.Count() / average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); @@ -127,7 +127,7 @@ void PerfDbReporter::ReportQPS(const ScenarioResult& result) const { perfDbClient_.setConfigs(result.client_config, result.server_config); } -void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) const { +void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) { auto qps = result.latencies.Count() / average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); @@ -139,7 +139,7 @@ void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) const { perfDbClient_.setConfigs(result.client_config, result.server_config); } -void PerfDbReporter::ReportLatency(const ScenarioResult& result) const { +void PerfDbReporter::ReportLatency(const ScenarioResult& result) { perfDbClient_.setLatencies(result.latencies.Percentile(50) / 1000, result.latencies.Percentile(90) / 1000, result.latencies.Percentile(95) / 1000, @@ -148,7 +148,7 @@ void PerfDbReporter::ReportLatency(const ScenarioResult& result) const { perfDbClient_.setConfigs(result.client_config, result.server_config); } -void PerfDbReporter::ReportTimes(const ScenarioResult& result) const { +void PerfDbReporter::ReportTimes(const ScenarioResult& result) { double serverSystemTime = 100.0 * sum(result.server_resources, [](ResourceUsage u) { return u.system_time; }) / sum(result.server_resources, @@ -171,7 +171,7 @@ void PerfDbReporter::ReportTimes(const ScenarioResult& result) const { perfDbClient_.setConfigs(result.client_config, result.server_config); } -void PerfDbReporter::SendData() const { +void PerfDbReporter::SendData() { //send data to performance database int dataState = perfDbClient_.sendData(access_token_, test_name_, sys_info_); diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 2ec7fe73f0d..bba26b990cf 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -109,7 +109,7 @@ class PerfDbReporter : public Reporter { public: PerfDbReporter(const string& name, const string& access_token, const string& test_name, const string& sys_info, const string& server_address) : Reporter(name), access_token_(access_token), test_name_(test_name), sys_info_(sys_info) { - perfDbClient.init(grpc::CreateChannel(server_address, grpc::InsecureCredentials(), ChannelArguments())); + perfDbClient_.init(grpc::CreateChannel(server_address, grpc::InsecureCredentials(), ChannelArguments())); } ~PerfDbReporter() { SendData(); }; @@ -118,11 +118,11 @@ class PerfDbReporter : public Reporter { std::string access_token_; std::string test_name_; std::string sys_info_; - void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE; - void ReportQPSPerCore(const ScenarioResult& result) const GRPC_OVERRIDE; - void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE; - void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; - void SendData() const; + void ReportQPS(const ScenarioResult& result) GRPC_OVERRIDE; + void ReportQPSPerCore(const ScenarioResult& result) GRPC_OVERRIDE; + void ReportLatency(const ScenarioResult& result) GRPC_OVERRIDE; + void ReportTimes(const ScenarioResult& result) GRPC_OVERRIDE; + void SendData(); }; } // namespace testing diff --git a/test/cpp/qps/run_perf_db_test.py b/test/cpp/qps/run_perf_db_test.py deleted file mode 100755 index a8e69ea1f02..00000000000 --- a/test/cpp/qps/run_perf_db_test.py +++ /dev/null @@ -1,289 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -import os -import sys -import re -import urllib2 -import urllib -import json -import time -import subprocess -import fnmatch -import argparse - -CLIENT_ID = '1018396037782-tv81fshn76nemr24uuhuginceb9hni2m.apps.googleusercontent.com' -CLIENT_SECRET = '_HGHXg4DAA59r4w4x8p6ARzD' -GRANT_TYPE = 'http://oauth.net/grant_type/device/1.0' -AUTH_TOKEN_LINK = 'https://www.googleapis.com/oauth2/v3/token' -GOOGLE_ACCOUNTS_LINK = 'https://accounts.google.com/o/oauth2/device/code' -USER_INFO_LINK = 'https://www.googleapis.com/oauth2/v1/userinfo' - -parser = argparse.ArgumentParser(description='Report metrics to performance database') -parser.add_argument('--test', type=str, help='Name of the test to be executed') -parser.add_argument('--email', type=str, help='Gmail address of the user') -parser.add_argument('--server_address', type=str, default='localhost:50052', help='Address of the performance database server') -parser.add_argument('--tokens_dir', type=str, default=os.path.expanduser('~')+'/.grpc/access_tokens', help='Path to the access tokens directory') - -# Fetches JSON reply object, given a url and parameters -def fetchJSON(url, paramDict): - if len(paramDict) == 0: - req = urllib2.Request(url) - else: - data = urllib.urlencode(paramDict) - req = urllib2.Request(url, data) - - try: - response = urllib2.urlopen(req) - result = response.read() - - except urllib2.HTTPError, error: - result = error.read() - - return result - -# Fetch user info; used to check if access token is valid -def getUserInfo(accessToken): - url = USER_INFO_LINK + '?access_token=' + accessToken - paramDict = {} - JSONBody = fetchJSON(url, paramDict) - data = json.loads(JSONBody) - - return data - -# Returns true if stored access token is valid -def isAccessTokenValid(accessToken): - data = getUserInfo(accessToken); - - if 'id' in data: - return True - else: - return False - -# Returns user id given a working access token -def getUserId(accessToken): - data = getUserInfo(accessToken) - - email = data['email'] - userId = getUserIdFromEmail(email) - - return userId - -# Extracts a unique user id from an email address -def getUserIdFromEmail(email): - email = email.split('@')[0].lower() # take username and convert to lower case - userId = re.sub('[.]', '', email) # remove periods - - return userId - -# Use an existing access token -def useAccessToken(userTokFile): - with open(userTokFile, "r") as data_file: - data = json.load(data_file) # load JSON data from file - accessToken = data["access_token"] - - # If access token has gone stale, refresh it - if not isAccessTokenValid(accessToken): - return refreshAccessToken(data["refresh_token"], userTokFile) - - return accessToken - -# refresh stale access token -def refreshAccessToken(refreshToken, userTokFile): - # Parameters for request - paramDict = {'refresh_token':refreshToken, 'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'grant_type':'refresh_token'} - # Fetch reply to request - JSONBody = fetchJSON(AUTH_TOKEN_LINK, paramDict) - data = json.loads(JSONBody) - - if not 'access_token' in data: - # Refresh token has gone stale, re-authentication required - return reauthenticate() - else: - # write fresh access token to tokens file - tokenData = {} - - with open(userTokFile, "r") as data_file: - tokenData = json.load(data_file) - - with open(userTokFile, "w") as data_file: - tokenData['access_token'] = data['access_token'] - json.dump(tokenData, data_file) - - # return fresh access token - return data['access_token'] - -def reauthenticate(tokensDir): - # Create folder if not created already - if not os.path.exists(tokensDir): - os.makedirs(tokensDir) - os.chmod(tokensDir, 0700) - - # Request parameters - paramDict = {'client_id':CLIENT_ID, 'scope':'email profile'} - JSONBody = fetchJSON(GOOGLE_ACCOUNTS_LINK, paramDict) - data = json.loads(JSONBody) - - print 'User authorization required\n' - print 'Please use the following code in you browser: ', data['user_code'] # Code to be entered by user in browser - print 'Verification URL: ', data['verification_url'] # Authentication link - print '\nAwaiting user authorization. May take a few more seconds after authorizing...\n' - - authData = {} - - while not 'access_token' in authData: - # Request parameters - authDict = {'client_id':CLIENT_ID, 'client_secret':CLIENT_SECRET, 'code':data['device_code'], 'grant_type':GRANT_TYPE} - JSONBody = fetchJSON(AUTH_TOKEN_LINK, authDict) - authData = json.loads(JSONBody) - # If server pinged too quickly, will get slowdown message; need to wait for specified interval - time.sleep(data['interval']) - - # File to write tokens - newUserTokFile = tokensDir + '/' + getUserId(authData['access_token']) - - # Write tokens to file - with open(newUserTokFile, "w") as data_file: - os.chmod(newUserTokFile, 0600) - json.dump(authData, data_file) - - # return working access token - return authData['access_token'] - -# Fetch a working access token given user entered email id; authntication may be required -def getAccessToken(email, tokensDir): - # Get unique user id from email address - userId = getUserIdFromEmail(email) - - # Token file - userTokFile = tokensDir + '/' + userId - - accessToken = '' - - if os.path.exists(userTokFile): - # File containing access token exists; unless refresh token has expired, user authentication will not be required - accessToken = useAccessToken(userTokFile) - else: - # User authentication required - accessToken = reauthenticate(tokensDir) - - return accessToken - -# If user has not entered full path to test, recursively searches for given test in parent folders -def findTestPath(test): - # If user entered full path to test, return it - if(os.path.isfile(test)): - return test - - testName = test.split('/')[-1] # Extract just test name - testPath = '' - - # Search for test - for root, dirnames, filenames in os.walk('../../../'): - for fileName in fnmatch.filter(filenames, testName): - testPath = os.path.join(root, fileName) - - return testPath - -def getSysInfo(): - # Fetch system information - sysInfo = os.popen('lscpu').readlines() - - NICs = os.popen('ifconfig | cut -c1-8 | sed \'/^\s*$/d\' | sort -u').readlines() - nicAddrs = os.popen('ifconfig | grep -oE "inet addr:([0-9]{1,3}\.){3}[0-9]{1,3}"').readlines() - - nicInfo = [] - - for i in range(0, len(NICs)): - NIC = NICs[i] - NIC = re.sub(r'[^\w]', '', NIC) - - ethtoolProcess = subprocess.Popen(["ethtool",NIC], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - ethtoolResult = ethtoolProcess.communicate()[0] - - ethtoolResultList = ethtoolResult.split('\n\t') - for ethtoolString in ethtoolResultList: - if ethtoolString.startswith('Speed'): - ethtoolString = ethtoolString.split(':')[1] - ethtoolString = ethtoolString.replace('Mb/s',' Mbps') - nicInfo.append('NIC ' + NIC + ' speed: ' + ethtoolString + '\n') - nicInfo.append(NIC + ' inet address: ' + nicAddrs[i].split(':')[1]) - - print 'Obtaining network info....' - tcp_rr_rate = str(os.popen('netperf -t TCP_RR -v 0').readlines()[1]) - print 'Network info obtained' - - nicInfo.append('TCP RR transmission rate per sec: ' + tcp_rr_rate + '\n') - sysInfo = sysInfo + nicInfo - - return sysInfo - -def main(argv): - args = parser.parse_args() - - tokensDir = args.tokens_dir - - try: - # Fetch working access token - accessToken = getAccessToken(args.email, tokensDir) - except AttributeError: - print '\nError: Please provide email address as an argument\n' - sys.exit(1) - except Exception, e: - print e, ' \nError in authentication\n' - sys.exit(1) - - # Address of the performance database server - serverAddress = args.server_address - - # Get path to test - try: - testPath = findTestPath(args.test) - except TypeError: - print '\nError: Please provide test name/path as argument\n' - sys.exit(1) - - # Get name of the test - testName = testPath.split('/')[-1] - - # Get the system information - sysInfo = getSysInfo() - - try: - print '\nBeginning test:\n' - # Run the test - subprocess.call([testPath, '--report_metrics_db=true', '--access_token='+accessToken, '--test_name='+testName, '--sys_info='+str(sysInfo).strip('[]'), '--server_address='+serverAddress]) - except OSError: - print 'Could not execute the test' - -if __name__ == "__main__": - main(sys.argv) \ No newline at end of file From 60111c0df846522c195a690abb01b6c536ff54a1 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 22 Jun 2015 13:02:33 -0700 Subject: [PATCH 23/63] updated syntax based errors --- test/cpp/qps/perf_db_client.cc | 26 ++++++++++++------------ test/cpp/qps/perf_db_client.h | 36 ++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/test/cpp/qps/perf_db_client.cc b/test/cpp/qps/perf_db_client.cc index 5b929087669..9ae8ba36fff 100644 --- a/test/cpp/qps/perf_db_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -38,37 +38,37 @@ namespace testing { //sets the client and server config information void PerfDbClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) { - clientConfig_ = clientConfig; - serverConfig_ = serverConfig; + this->clientConfig_ = clientConfig; + this->serverConfig_ = serverConfig; } //sets the QPS void PerfDbClient::setQPS(double QPS) { - QPS_ = QPS; + this->QPS_ = QPS; } //sets the QPS per core void PerfDbClient::setQPSPerCore(double QPSPerCore) { - QPSPerCore_ = QPSPerCore; + this->QPSPerCore_ = QPSPerCore; } //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency void PerfDbClient::setLatencies(double percentileLatency50, double percentileLatency90, double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) { - percentileLatency50_ = percentileLatency50; - percentileLatency90_ = percentileLatency90; - percentileLatency95_ = percentileLatency95; - percentileLatency99_ = percentileLatency99; - percentileLatency99Point9_ = percentileLatency99Point9; + this->percentileLatency50_ = percentileLatency50; + this->percentileLatency90_ = percentileLatency90; + this->percentileLatency95_ = percentileLatency95; + this->percentileLatency99_ = percentileLatency99; + this->percentileLatency99Point9_ = percentileLatency99Point9; } //sets the server and client, user and system times void PerfDbClient::setTimes(double serverSystemTime, double serverUserTime, double clientSystemTime, double clientUserTime) { - serverSystemTime_ = serverSystemTime; - serverUserTime_ = serverUserTime; - clientSystemTime_ = clientSystemTime; - clientUserTime_ = clientUserTime; + this->serverSystemTime_ = serverSystemTime; + this->serverUserTime_ = serverUserTime; + this->clientSystemTime_ = clientSystemTime; + this->clientUserTime_ = clientUserTime; } //sends the data to the performancew database server diff --git a/test/cpp/qps/perf_db_client.h b/test/cpp/qps/perf_db_client.h index 058cd6a2070..c19442f35e5 100644 --- a/test/cpp/qps/perf_db_client.h +++ b/test/cpp/qps/perf_db_client.h @@ -52,7 +52,19 @@ namespace testing { //Manages data sending to performance database server class PerfDbClient { public: - PerfDbClient() {} + PerfDbClient() { + QPS_ = DBL_MIN; + QPSPerCore_ = DBL_MIN; + percentileLatency50_ = DBL_MIN; + percentileLatency90_ = DBL_MIN; + percentileLatency95_ = DBL_MIN; + percentileLatency99_ = DBL_MIN; + percentileLatency99Point9_ = DBL_MIN; + serverSystemTime_ = DBL_MIN; + serverUserTime_ = DBL_MIN; + clientSystemTime_ = DBL_MIN; + clientUserTime_ = DBL_MIN; + } void init(std::shared_ptr channel) { stub_ = PerfDbTransfer::NewStub(channel); } @@ -82,17 +94,17 @@ private: std::unique_ptr stub_; ClientConfig clientConfig_; ServerConfig serverConfig_; - double QPS_ = DBL_MIN; - double QPSPerCore_ = DBL_MIN; - double percentileLatency50_ = DBL_MIN; - double percentileLatency90_ = DBL_MIN; - double percentileLatency95_ = DBL_MIN; - double percentileLatency99_ = DBL_MIN; - double percentileLatency99Point9_ = DBL_MIN; - double serverSystemTime_ = DBL_MIN; - double serverUserTime_ = DBL_MIN; - double clientSystemTime_ = DBL_MIN; - double clientUserTime_ = DBL_MIN; + double QPS_; + double QPSPerCore_; + double percentileLatency50_; + double percentileLatency90_; + double percentileLatency95_; + double percentileLatency99_; + double percentileLatency99Point9_; + double serverSystemTime_; + double serverUserTime_; + double clientSystemTime_; + double clientUserTime_; }; } //namespace testing From a4de070c69d81b1a79e7727a6b342a6b248daaa6 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 22 Jun 2015 13:05:26 -0700 Subject: [PATCH 24/63] Further corrections to syntax --- test/cpp/qps/perf_db_client.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/cpp/qps/perf_db_client.cc b/test/cpp/qps/perf_db_client.cc index 9ae8ba36fff..0905cde11a0 100644 --- a/test/cpp/qps/perf_db_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -82,23 +82,23 @@ int PerfDbClient::sendData(std::string access_token, std::string test_name, std: singleUserRecordRequest.set_sys_info(sys_info); //setting configs - *(singleUserRecordRequest.mutable_client_config()) = clientConfig_; - *(singleUserRecordRequest.mutable_server_config()) = serverConfig_; + *(singleUserRecordRequest.mutable_client_config()) = this->clientConfig_; + *(singleUserRecordRequest.mutable_server_config()) = this->serverConfig_; Metrics* metrics = singleUserRecordRequest.mutable_metrics(); //setting metrcs in data record request - if(QPS_ != DBL_MIN) metrics->set_qps(QPS_); - if(QPSPerCore_ != DBL_MIN) metrics->set_qps_per_core(QPSPerCore_); - if(percentileLatency50_ != DBL_MIN) metrics->set_perc_lat_50(percentileLatency50_); - if(percentileLatency90_ != DBL_MIN) metrics->set_perc_lat_90(percentileLatency90_); - if(percentileLatency95_ != DBL_MIN) metrics->set_perc_lat_95(percentileLatency95_); - if(percentileLatency99_ != DBL_MIN) metrics->set_perc_lat_99(percentileLatency99_); - if(percentileLatency99Point9_ != DBL_MIN) metrics->set_perc_lat_99_point_9(percentileLatency99Point9_); - if(serverSystemTime_ != DBL_MIN) metrics->set_server_system_time(serverSystemTime_); - if(serverUserTime_ != DBL_MIN) metrics->set_server_user_time(serverUserTime_); - if(clientSystemTime_ != DBL_MIN) metrics->set_client_system_time(clientSystemTime_); - if(clientUserTime_ != DBL_MIN) metrics->set_client_user_time(clientUserTime_); + if(QPS_ != DBL_MIN) metrics->set_qps(this->QPS_); + if(QPSPerCore_ != DBL_MIN) metrics->set_qps_per_core(this->QPSPerCore_); + if(percentileLatency50_ != DBL_MIN) metrics->set_perc_lat_50(this->percentileLatency50_); + if(percentileLatency90_ != DBL_MIN) metrics->set_perc_lat_90(this->percentileLatency90_); + if(percentileLatency95_ != DBL_MIN) metrics->set_perc_lat_95(this->percentileLatency95_); + if(percentileLatency99_ != DBL_MIN) metrics->set_perc_lat_99(this->percentileLatency99_); + if(percentileLatency99Point9_ != DBL_MIN) metrics->set_perc_lat_99_point_9(this->percentileLatency99Point9_); + if(serverSystemTime_ != DBL_MIN) metrics->set_server_system_time(this->serverSystemTime_); + if(serverUserTime_ != DBL_MIN) metrics->set_server_user_time(this->serverUserTime_); + if(clientSystemTime_ != DBL_MIN) metrics->set_client_system_time(this->clientSystemTime_); + if(clientUserTime_ != DBL_MIN) metrics->set_client_user_time(this->clientUserTime_); SingleUserRecordReply singleUserRecordReply; ClientContext context; From ca241639c65cd5491434bfe0dc021e38b40008c5 Mon Sep 17 00:00:00 2001 From: sidrakesh93 Date: Mon, 22 Jun 2015 14:26:41 -0700 Subject: [PATCH 25/63] Update perf_db_client.cc --- test/cpp/qps/perf_db_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/perf_db_client.cc b/test/cpp/qps/perf_db_client.cc index 0905cde11a0..f514408c14c 100644 --- a/test/cpp/qps/perf_db_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -111,4 +111,4 @@ int PerfDbClient::sendData(std::string access_token, std::string test_name, std: } } } //testing -} //grpc \ No newline at end of file +} //grpc From df77c580dc62f17d848f1899e72501d60b098320 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Tue, 23 Jun 2015 16:34:18 -0700 Subject: [PATCH 26/63] Added support for passing a tag with the test --- test/cpp/qps/perf_db.proto | 14 ++++++++------ test/cpp/qps/perf_db_client.cc | 3 ++- test/cpp/qps/perf_db_client.h | 2 +- test/cpp/qps/report.cc | 2 +- test/cpp/qps/report.h | 6 ++++-- test/cpp/util/benchmark_config.cc | 5 ++++- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/test/cpp/qps/perf_db.proto b/test/cpp/qps/perf_db.proto index 50070fda5b6..7f4a460812f 100644 --- a/test/cpp/qps/perf_db.proto +++ b/test/cpp/qps/perf_db.proto @@ -62,9 +62,10 @@ message DataDetails { string timestamp = 1; string test_name = 2; string sys_info = 3; - Metrics metrics = 4; - ClientConfig client_config = 5; - ServerConfig server_config = 6; + string tag = 4; + Metrics metrics = 5; + ClientConfig client_config = 6; + ServerConfig server_config = 7; } //User details @@ -93,9 +94,10 @@ message SingleUserRecordRequest { string access_token = 1; string test_name = 2; string sys_info = 3; - Metrics metrics = 4; - ClientConfig client_config = 5; - ServerConfig server_config = 6; + string tag = 4; + Metrics metrics = 5; + ClientConfig client_config = 6; + ServerConfig server_config = 7; } //Reply to request for storing single user's data diff --git a/test/cpp/qps/perf_db_client.cc b/test/cpp/qps/perf_db_client.cc index 0905cde11a0..c3a170b398d 100644 --- a/test/cpp/qps/perf_db_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -72,7 +72,7 @@ void PerfDbClient::setTimes(double serverSystemTime, double serverUserTime, } //sends the data to the performancew database server -int PerfDbClient::sendData(std::string access_token, std::string test_name, std::string sys_info) { +int PerfDbClient::sendData(std::string access_token, std::string test_name, std::string sys_info, std::string tag) { //Data record request object SingleUserRecordRequest singleUserRecordRequest; @@ -80,6 +80,7 @@ int PerfDbClient::sendData(std::string access_token, std::string test_name, std: singleUserRecordRequest.set_access_token(access_token); singleUserRecordRequest.set_test_name(test_name); singleUserRecordRequest.set_sys_info(sys_info); + singleUserRecordRequest.set_tag(tag); //setting configs *(singleUserRecordRequest.mutable_client_config()) = this->clientConfig_; diff --git a/test/cpp/qps/perf_db_client.h b/test/cpp/qps/perf_db_client.h index c19442f35e5..be4766ab055 100644 --- a/test/cpp/qps/perf_db_client.h +++ b/test/cpp/qps/perf_db_client.h @@ -88,7 +88,7 @@ public: double clientSystemTime, double clientUserTime); //sends the data to the performancew database server - int sendData(std::string access_token, std::string test_name, std::string sys_info); + int sendData(std::string access_token, std::string test_name, std::string sys_info, std::string tag); private: std::unique_ptr stub_; diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 59c0da31fb9..7167d4e336a 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -173,7 +173,7 @@ void PerfDbReporter::ReportTimes(const ScenarioResult& result) { void PerfDbReporter::SendData() { //send data to performance database - int dataState = perfDbClient_.sendData(access_token_, test_name_, sys_info_); + int dataState = perfDbClient_.sendData(access_token_, test_name_, sys_info_, tag_); //check state of data sending switch(dataState) { diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index bba26b990cf..565590649ab 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -107,8 +107,9 @@ class GprLogReporter : public Reporter { /** Reporter for performance database tool */ class PerfDbReporter : public Reporter { public: - PerfDbReporter(const string& name, const string& access_token, const string& test_name, const string& sys_info, const string& server_address) - : Reporter(name), access_token_(access_token), test_name_(test_name), sys_info_(sys_info) { + PerfDbReporter(const string& name, const string& access_token, const string& test_name, + const string& sys_info, const string& server_address, const string& tag) + : Reporter(name), access_token_(access_token), test_name_(test_name), sys_info_(sys_info), tag_(tag) { perfDbClient_.init(grpc::CreateChannel(server_address, grpc::InsecureCredentials(), ChannelArguments())); } ~PerfDbReporter() { SendData(); }; @@ -118,6 +119,7 @@ class PerfDbReporter : public Reporter { std::string access_token_; std::string test_name_; std::string sys_info_; + std::string tag_; void ReportQPS(const ScenarioResult& result) GRPC_OVERRIDE; void ReportQPSPerCore(const ScenarioResult& result) GRPC_OVERRIDE; void ReportLatency(const ScenarioResult& result) GRPC_OVERRIDE; diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc index f8cfabd0d68..030cb28c32f 100644 --- a/test/cpp/util/benchmark_config.cc +++ b/test/cpp/util/benchmark_config.cc @@ -47,6 +47,8 @@ DEFINE_string(sys_info, "", "System information"); DEFINE_string(server_address, "localhost:50052", "Address of the performance database server"); +DEFINE_string(tag, "", "Optional tag for the test"); + // In some distros, gflags is in the namespace google, and in some others, // in gflags. This hack is enabling us to find both. namespace google {} @@ -69,7 +71,8 @@ static std::shared_ptr InitBenchmarkReporters() { } if(FLAGS_report_metrics_db) { composite_reporter->add( - std::unique_ptr(new PerfDbReporter("PerfDbReporter", FLAGS_access_token, FLAGS_test_name, FLAGS_sys_info, FLAGS_server_address))); + std::unique_ptr(new PerfDbReporter("PerfDbReporter", FLAGS_access_token, FLAGS_test_name, + FLAGS_sys_info, FLAGS_server_address, FLAGS_tag))); } return std::shared_ptr(composite_reporter); From c5444224538165ccabd005c5f7f50a4f4627a32a Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 24 Jun 2015 14:08:07 -0700 Subject: [PATCH 27/63] Added zlib as a linker dep. for Ruby --- src/ruby/ext/grpc/extconf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 0ff8bb9aa7a..6dd02344892 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -89,7 +89,7 @@ $CFLAGS << ' -Wno-return-type ' $CFLAGS << ' -Wall ' $CFLAGS << ' -pedantic ' -$LDFLAGS << ' -lgrpc -lgpr -ldl' +$LDFLAGS << ' -lgrpc -lgpr -lz -ldl' crash('need grpc lib') unless have_library('grpc', 'grpc_channel_destroy') have_library('grpc', 'grpc_channel_destroy') From 268e59063a719856eec942c7eace029f34ca8432 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 29 Jun 2015 12:20:43 -0700 Subject: [PATCH 28/63] Adding required changes --- test/cpp/qps/perf_db.proto | 65 +++------------------- test/cpp/qps/perf_db_client.cc | 98 +++++++++++++++++++++------------- test/cpp/qps/perf_db_client.h | 48 +++++++++-------- test/cpp/qps/report.cc | 72 ++++++++++++------------- test/cpp/qps/report.h | 16 ++++-- 5 files changed, 138 insertions(+), 161 deletions(-) diff --git a/test/cpp/qps/perf_db.proto b/test/cpp/qps/perf_db.proto index 7f4a460812f..7a550f83c65 100644 --- a/test/cpp/qps/perf_db.proto +++ b/test/cpp/qps/perf_db.proto @@ -35,14 +35,12 @@ package grpc.testing; service PerfDbTransfer { // Sends client info - rpc RecordSingleClientData (SingleUserRecordRequest) returns (SingleUserRecordReply) {} - - rpc RetrieveSingleUserData (SingleUserRetrieveRequest) returns (SingleUserRetrieveReply) {} - - rpc RetrieveAllUsersData (AllUsersRetrieveRequest) returns (AllUsersRetrieveReply) {} + rpc RecordSingleClientData(SingleUserRecordRequest) + returns (SingleUserRecordReply) { + } } -//Metrics to be stored +// Metrics to be stored message Metrics { double qps = 1; double qps_per_core = 2; @@ -57,39 +55,7 @@ message Metrics { double client_user_time = 11; } -//Timestamped details -message DataDetails { - string timestamp = 1; - string test_name = 2; - string sys_info = 3; - string tag = 4; - Metrics metrics = 5; - ClientConfig client_config = 6; - ServerConfig server_config = 7; -} - -//User details -message UserDetails { - string id = 1; - string email = 2; - bool verified_email = 3; - string name = 4; - string given_name = 5; - string family_name = 6; - string link = 7; - string picture = 8; - string gender = 9; - string locale = 10; - string hd = 11; -} - -//Stored to database -message SingleUserDetails { - repeated DataDetails data_details = 1; - UserDetails user_details = 2; -} - -//Request for storing a single user's data +// Request for storing a single user's data message SingleUserRecordRequest { string access_token = 1; string test_name = 2; @@ -100,25 +66,6 @@ message SingleUserRecordRequest { ServerConfig server_config = 7; } -//Reply to request for storing single user's data +// Reply to request for storing single user's data message SingleUserRecordReply { } - -//Request for retrieving single user's data -message SingleUserRetrieveRequest { - string user_id = 1; -} - -//Reply for request to retrieve single user's data -message SingleUserRetrieveReply { - SingleUserDetails details = 1; -} - -//Request for retrieving all users' data -message AllUsersRetrieveReply { - repeated SingleUserDetails user_data = 1; -} - -//Reply to request for retrieving all users' data -message AllUsersRetrieveRequest { -} diff --git a/test/cpp/qps/perf_db_client.cc b/test/cpp/qps/perf_db_client.cc index 65569e50778..c4ee0ae5b8d 100644 --- a/test/cpp/qps/perf_db_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -36,25 +36,27 @@ namespace grpc { namespace testing { -//sets the client and server config information -void PerfDbClient::setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig) { +// sets the client and server config information +void PerfDbClient::setConfigs(const ClientConfig& clientConfig, + const ServerConfig& serverConfig) { this->clientConfig_ = clientConfig; this->serverConfig_ = serverConfig; } -//sets the QPS -void PerfDbClient::setQPS(double QPS) { - this->QPS_ = QPS; -} +// sets the QPS +void PerfDbClient::setQPS(double QPS) { this->QPS_ = QPS; } -//sets the QPS per core +// sets the QPS per core void PerfDbClient::setQPSPerCore(double QPSPerCore) { this->QPSPerCore_ = QPSPerCore; } -//sets the 50th, 90th, 95th, 99th and 99.9th percentile latency -void PerfDbClient::setLatencies(double percentileLatency50, double percentileLatency90, - double percentileLatency95, double percentileLatency99, double percentileLatency99Point9) { +// sets the 50th, 90th, 95th, 99th and 99.9th percentile latency +void PerfDbClient::setLatencies(double percentileLatency50, + double percentileLatency90, + double percentileLatency95, + double percentileLatency99, + double percentileLatency99Point9) { this->percentileLatency50_ = percentileLatency50; this->percentileLatency90_ = percentileLatency90; this->percentileLatency95_ = percentileLatency95; @@ -62,54 +64,78 @@ void PerfDbClient::setLatencies(double percentileLatency50, double percentileLat this->percentileLatency99Point9_ = percentileLatency99Point9; } -//sets the server and client, user and system times -void PerfDbClient::setTimes(double serverSystemTime, double serverUserTime, - double clientSystemTime, double clientUserTime) { +// sets the server and client, user and system times +void PerfDbClient::setTimes(double serverSystemTime, double serverUserTime, + double clientSystemTime, double clientUserTime) { this->serverSystemTime_ = serverSystemTime; this->serverUserTime_ = serverUserTime; this->clientSystemTime_ = clientSystemTime; this->clientUserTime_ = clientUserTime; } -//sends the data to the performancew database server -int PerfDbClient::sendData(std::string access_token, std::string test_name, std::string sys_info, std::string tag) { - //Data record request object +// sends the data to the performance database server +bool PerfDbClient::sendData(std::string access_token, std::string test_name, + std::string sys_info, std::string tag) { + // Data record request object SingleUserRecordRequest singleUserRecordRequest; - //setting access token, name of the test and the system information + // setting access token, name of the test and the system information singleUserRecordRequest.set_access_token(access_token); singleUserRecordRequest.set_test_name(test_name); singleUserRecordRequest.set_sys_info(sys_info); singleUserRecordRequest.set_tag(tag); - //setting configs + // setting configs *(singleUserRecordRequest.mutable_client_config()) = this->clientConfig_; *(singleUserRecordRequest.mutable_server_config()) = this->serverConfig_; - + Metrics* metrics = singleUserRecordRequest.mutable_metrics(); - //setting metrcs in data record request - if(QPS_ != DBL_MIN) metrics->set_qps(this->QPS_); - if(QPSPerCore_ != DBL_MIN) metrics->set_qps_per_core(this->QPSPerCore_); - if(percentileLatency50_ != DBL_MIN) metrics->set_perc_lat_50(this->percentileLatency50_); - if(percentileLatency90_ != DBL_MIN) metrics->set_perc_lat_90(this->percentileLatency90_); - if(percentileLatency95_ != DBL_MIN) metrics->set_perc_lat_95(this->percentileLatency95_); - if(percentileLatency99_ != DBL_MIN) metrics->set_perc_lat_99(this->percentileLatency99_); - if(percentileLatency99Point9_ != DBL_MIN) metrics->set_perc_lat_99_point_9(this->percentileLatency99Point9_); - if(serverSystemTime_ != DBL_MIN) metrics->set_server_system_time(this->serverSystemTime_); - if(serverUserTime_ != DBL_MIN) metrics->set_server_user_time(this->serverUserTime_); - if(clientSystemTime_ != DBL_MIN) metrics->set_client_system_time(this->clientSystemTime_); - if(clientUserTime_ != DBL_MIN) metrics->set_client_user_time(this->clientUserTime_); + // setting metrcs in data record request + if (QPS_ != DBL_MIN) { + metrics->set_qps(this->QPS_); + } + if (QPSPerCore_ != DBL_MIN) { + metrics->set_qps_per_core(this->QPSPerCore_); + } + if (percentileLatency50_ != DBL_MIN) { + metrics->set_perc_lat_50(this->percentileLatency50_); + } + if (percentileLatency90_ != DBL_MIN) { + metrics->set_perc_lat_90(this->percentileLatency90_); + } + if (percentileLatency95_ != DBL_MIN) { + metrics->set_perc_lat_95(this->percentileLatency95_); + } + if (percentileLatency99_ != DBL_MIN) { + metrics->set_perc_lat_99(this->percentileLatency99_); + } + if (percentileLatency99Point9_ != DBL_MIN) { + metrics->set_perc_lat_99_point_9(this->percentileLatency99Point9_); + } + if (serverSystemTime_ != DBL_MIN) { + metrics->set_server_system_time(this->serverSystemTime_); + } + if (serverUserTime_ != DBL_MIN) { + metrics->set_server_user_time(this->serverUserTime_); + } + if (clientSystemTime_ != DBL_MIN) { + metrics->set_client_system_time(this->clientSystemTime_); + } + if (clientUserTime_ != DBL_MIN) { + metrics->set_client_user_time(this->clientUserTime_); + } SingleUserRecordReply singleUserRecordReply; ClientContext context; - Status status = stub_->RecordSingleClientData(&context, singleUserRecordRequest, &singleUserRecordReply); + Status status = stub_->RecordSingleClientData( + &context, singleUserRecordRequest, &singleUserRecordReply); if (status.ok()) { - return 1; //data sent to database successfully + return true; // data sent to database successfully } else { - return -1; //error in data sending + return false; // error in data sending } } -} //testing -} //grpc +} // testing +} // grpc diff --git a/test/cpp/qps/perf_db_client.h b/test/cpp/qps/perf_db_client.h index be4766ab055..b1ba98831cc 100644 --- a/test/cpp/qps/perf_db_client.h +++ b/test/cpp/qps/perf_db_client.h @@ -45,13 +45,12 @@ #include #include "test/cpp/qps/perf_db.grpc.pb.h" - -namespace grpc{ +namespace grpc { namespace testing { -//Manages data sending to performance database server +// Manages data sending to performance database server class PerfDbClient { -public: + public: PerfDbClient() { QPS_ = DBL_MIN; QPSPerCore_ = DBL_MIN; @@ -65,32 +64,37 @@ public: clientSystemTime_ = DBL_MIN; clientUserTime_ = DBL_MIN; } - - void init(std::shared_ptr channel) { stub_ = PerfDbTransfer::NewStub(channel); } + + void init(std::shared_ptr channel) { + stub_ = PerfDbTransfer::NewStub(channel); + } ~PerfDbClient() {} - //sets the client and server config information - void setConfigs(const ClientConfig& clientConfig, const ServerConfig& serverConfig); - - //sets the QPS + // sets the client and server config information + void setConfigs(const ClientConfig& clientConfig, + const ServerConfig& serverConfig); + + // sets the QPS void setQPS(double QPS); - //sets the QPS per core + // sets the QPS per core void setQPSPerCore(double QPSPerCore); - //sets the 50th, 90th, 95th, 99th and 99.9th percentile latency + // sets the 50th, 90th, 95th, 99th and 99.9th percentile latency void setLatencies(double percentileLatency50, double percentileLatency90, - double percentileLatency95, double percentileLatency99, double percentileLatency99Point9); + double percentileLatency95, double percentileLatency99, + double percentileLatency99Point9); - //sets the server and client, user and system times - void setTimes(double serverSystemTime, double serverUserTime, - double clientSystemTime, double clientUserTime); + // sets the server and client, user and system times + void setTimes(double serverSystemTime, double serverUserTime, + double clientSystemTime, double clientUserTime); - //sends the data to the performancew database server - int sendData(std::string access_token, std::string test_name, std::string sys_info, std::string tag); + // sends the data to the performance database server + bool sendData(std::string access_token, std::string test_name, + std::string sys_info, std::string tag); -private: + private: std::unique_ptr stub_; ClientConfig clientConfig_; ServerConfig serverConfig_; @@ -107,7 +111,5 @@ private: double clientUserTime_; }; -} //namespace testing -} //namespace grpc - - +} // namespace testing +} // namespace grpc diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 7167d4e336a..d8041adca40 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -67,7 +67,6 @@ void CompositeReporter::ReportTimes(const ScenarioResult& result) { } } - void GprLogReporter::ReportQPS(const ScenarioResult& result) { gpr_log(GPR_INFO, "QPS: %.1f", result.latencies.Count() / @@ -76,10 +75,9 @@ void GprLogReporter::ReportQPS(const ScenarioResult& result) { } void GprLogReporter::ReportQPSPerCore(const ScenarioResult& result) { - auto qps = - result.latencies.Count() / - average(result.client_resources, - [](ResourceUsage u) { return u.wall_time; }); + auto qps = result.latencies.Count() / + average(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); gpr_log(GPR_INFO, "QPS: %.1f (%.1f/server core)", qps, qps / result.server_config.threads()); @@ -120,8 +118,8 @@ void GprLogReporter::ReportTimes(const ScenarioResult& result) { void PerfDbReporter::ReportQPS(const ScenarioResult& result) { auto qps = result.latencies.Count() / - average(result.client_resources, - [](ResourceUsage u) { return u.wall_time; }); + average(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); perfDbClient_.setQPS(qps); perfDbClient_.setConfigs(result.client_config, result.server_config); @@ -129,8 +127,8 @@ void PerfDbReporter::ReportQPS(const ScenarioResult& result) { void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) { auto qps = result.latencies.Count() / - average(result.client_resources, - [](ResourceUsage u) { return u.wall_time; }); + average(result.client_resources, + [](ResourceUsage u) { return u.wall_time; }); auto qpsPerCore = qps / result.server_config.threads(); @@ -141,48 +139,46 @@ void PerfDbReporter::ReportQPSPerCore(const ScenarioResult& result) { void PerfDbReporter::ReportLatency(const ScenarioResult& result) { perfDbClient_.setLatencies(result.latencies.Percentile(50) / 1000, - result.latencies.Percentile(90) / 1000, - result.latencies.Percentile(95) / 1000, - result.latencies.Percentile(99) / 1000, - result.latencies.Percentile(99.9) / 1000); + result.latencies.Percentile(90) / 1000, + result.latencies.Percentile(95) / 1000, + result.latencies.Percentile(99) / 1000, + result.latencies.Percentile(99.9) / 1000); perfDbClient_.setConfigs(result.client_config, result.server_config); } void PerfDbReporter::ReportTimes(const ScenarioResult& result) { - double serverSystemTime = 100.0 * sum(result.server_resources, + double serverSystemTime = + 100.0 * sum(result.server_resources, [](ResourceUsage u) { return u.system_time; }) / - sum(result.server_resources, - [](ResourceUsage u) { return u.wall_time; }); - double serverUserTime = 100.0 * sum(result.server_resources, + sum(result.server_resources, [](ResourceUsage u) { return u.wall_time; }); + double serverUserTime = + 100.0 * sum(result.server_resources, [](ResourceUsage u) { return u.user_time; }) / - sum(result.server_resources, - [](ResourceUsage u) { return u.wall_time; }); - double clientSystemTime = 100.0 * sum(result.client_resources, + sum(result.server_resources, [](ResourceUsage u) { return u.wall_time; }); + double clientSystemTime = + 100.0 * sum(result.client_resources, [](ResourceUsage u) { return u.system_time; }) / - sum(result.client_resources, - [](ResourceUsage u) { return u.wall_time; }); - double clientUserTime = 100.0 * sum(result.client_resources, + sum(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); + double clientUserTime = + 100.0 * sum(result.client_resources, [](ResourceUsage u) { return u.user_time; }) / - sum(result.client_resources, - [](ResourceUsage u) { return u.wall_time; }); + sum(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); - perfDbClient_.setTimes(serverSystemTime, serverUserTime, - clientSystemTime, clientUserTime); + perfDbClient_.setTimes(serverSystemTime, serverUserTime, clientSystemTime, + clientUserTime); perfDbClient_.setConfigs(result.client_config, result.server_config); } void PerfDbReporter::SendData() { - //send data to performance database - int dataState = perfDbClient_.sendData(access_token_, test_name_, sys_info_, tag_); - - //check state of data sending - switch(dataState) { - case 1: - gpr_log(GPR_INFO, "Data sent to performance database successfully"); - break; - case -1: - gpr_log(GPR_INFO, "Data could not be sent to performance database"); - break; + // send data to performance database + bool dataState = + perfDbClient_.sendData(access_token_, test_name_, sys_info_, tag_); + + // check state of data sending + if (dataState) { + gpr_log(GPR_INFO, "Data sent to performance database successfully"); + } else { + gpr_log(GPR_INFO, "Data could not be sent to performance database"); } } diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 565590649ab..330d4ebd295 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -107,12 +107,18 @@ class GprLogReporter : public Reporter { /** Reporter for performance database tool */ class PerfDbReporter : public Reporter { public: - PerfDbReporter(const string& name, const string& access_token, const string& test_name, - const string& sys_info, const string& server_address, const string& tag) - : Reporter(name), access_token_(access_token), test_name_(test_name), sys_info_(sys_info), tag_(tag) { - perfDbClient_.init(grpc::CreateChannel(server_address, grpc::InsecureCredentials(), ChannelArguments())); + PerfDbReporter(const string& name, const string& access_token, + const string& test_name, const string& sys_info, + const string& server_address, const string& tag) + : Reporter(name), + access_token_(access_token), + test_name_(test_name), + sys_info_(sys_info), + tag_(tag) { + perfDbClient_.init(grpc::CreateChannel( + server_address, grpc::InsecureCredentials(), ChannelArguments())); } - ~PerfDbReporter() { SendData(); }; + ~PerfDbReporter() GRPC_OVERRIDE { SendData(); }; private: PerfDbClient perfDbClient_; From 25481f72c1d519136b7fdd13f906f6ecaa9ee268 Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 29 Jun 2015 13:27:23 -0700 Subject: [PATCH 29/63] Resolving merge conflict --- Makefile | 71 +----------------------- build.json | 15 ----- tools/run_tests/sources_and_headers.json | 3 + 3 files changed, 5 insertions(+), 84 deletions(-) diff --git a/Makefile b/Makefile index b1da79ea4c8..195793342c6 100644 --- a/Makefile +++ b/Makefile @@ -3557,63 +3557,6 @@ endif endif -<<<<<<< HEAD -LIBGRPC++_BENCHMARK_CONFIG_SRC = \ - $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc \ - $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc \ - test/cpp/qps/perf_db_client.cc \ - test/cpp/qps/report.cc \ - test/cpp/util/benchmark_config.cc \ - - -LIBGRPC++_BENCHMARK_CONFIG_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_BENCHMARK_CONFIG_SRC)))) - -ifeq ($(NO_SECURE),true) - -# You can't build secure libraries if you don't have OpenSSL with ALPN. - -$(LIBDIR)/$(CONFIG)/libgrpc++_benchmark_config.a: openssl_dep_error - - -else - -ifeq ($(NO_PROTOBUF),true) - -# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay. - -$(LIBDIR)/$(CONFIG)/libgrpc++_benchmark_config.a: protobuf_dep_error - - -else - -$(LIBDIR)/$(CONFIG)/libgrpc++_benchmark_config.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(PROTOBUF_DEP) $(LIBGRPC++_BENCHMARK_CONFIG_OBJS) - $(E) "[AR] Creating $@" - $(Q) mkdir -p `dirname $@` - $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc++_benchmark_config.a - $(Q) $(AR) rcs $(LIBDIR)/$(CONFIG)/libgrpc++_benchmark_config.a $(LIBGRPC++_BENCHMARK_CONFIG_OBJS) -ifeq ($(SYSTEM),Darwin) - $(Q) ranlib $(LIBDIR)/$(CONFIG)/libgrpc++_benchmark_config.a -endif - - - - -endif - -endif - -ifneq ($(NO_SECURE),true) -ifneq ($(NO_DEPS),true) --include $(LIBGRPC++_BENCHMARK_CONFIG_OBJS:.o=.dep) -endif -endif -$(OBJDIR)/$(CONFIG)/test/cpp/qps/perf_db_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc - - -======= ->>>>>>> a6de02f5ef15646c12f495f11c410326d34e5dfe LIBGRPC++_TEST_CONFIG_SRC = \ test/cpp/util/test_config.cc \ @@ -4180,26 +4123,16 @@ ifneq ($(NO_DEPS),true) -include $(LIBQPS_OBJS:.o=.dep) endif endif -<<<<<<< HEAD $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/perf_db_client.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/server_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc $(OBJDIR)/$(CONFIG)/test/cpp/qps/timer.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc -======= -$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/client_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/driver.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_worker.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/report.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_async.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/server_sync.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/qps/timer.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc -$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc ->>>>>>> a6de02f5ef15646c12f495f11c410326d34e5dfe +$(OBJDIR)/$(CONFIG)/test/cpp/util/benchmark_config.o: $(GENDIR)/test/cpp/qps/qpstest.pb.cc $(GENDIR)/test/cpp/qps/qpstest.grpc.pb.cc $(GENDIR)/test/cpp/qps/perf_db.pb.cc $(GENDIR)/test/cpp/qps/perf_db.grpc.pb.cc LIBGRPC_CSHARP_EXT_SRC = \ diff --git a/build.json b/build.json index 39f6158d0c6..7e37bc4e29d 100644 --- a/build.json +++ b/build.json @@ -556,21 +556,6 @@ "vs_project_guid": "{C187A093-A0FE-489D-A40A-6E33DE0F9FEB}" }, { -<<<<<<< HEAD - "name": "grpc++_benchmark_config", - "build": "private", - "language": "c++", - "src": [ - "test/cpp/qps/qpstest.proto", - "test/cpp/qps/perf_db.proto", - "test/cpp/qps/perf_db_client.cc", - "test/cpp/qps/report.cc", - "test/cpp/util/benchmark_config.cc" - ] - }, - { -======= ->>>>>>> a6de02f5ef15646c12f495f11c410326d34e5dfe "name": "grpc++_test_config", "build": "private", "language": "c++", diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index eb2514fa46d..13d77d571bf 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -9779,6 +9779,8 @@ "test/cpp/qps/driver.h", "test/cpp/qps/histogram.h", "test/cpp/qps/interarrival.h", + "test/cpp/qps/perf_db.grpc.pb.h", + "test/cpp/qps/perf_db.pb.h", "test/cpp/qps/qps_worker.h", "test/cpp/qps/qpstest.grpc.pb.h", "test/cpp/qps/qpstest.pb.h", @@ -9798,6 +9800,7 @@ "test/cpp/qps/driver.h", "test/cpp/qps/histogram.h", "test/cpp/qps/interarrival.h", + "test/cpp/qps/perf_db_client.cc", "test/cpp/qps/qps_worker.cc", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.cc", From b713546e4daa47f9b6555e6d1db71c88dc1be1fd Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Mon, 29 Jun 2015 14:14:37 -0700 Subject: [PATCH 30/63] Resolving dependency error in Jenkins --- build.json | 1 + tools/run_tests/sources_and_headers.json | 2 ++ 2 files changed, 3 insertions(+) diff --git a/build.json b/build.json index 7e37bc4e29d..56c39045625 100644 --- a/build.json +++ b/build.json @@ -748,6 +748,7 @@ "test/cpp/qps/driver.h", "test/cpp/qps/histogram.h", "test/cpp/qps/interarrival.h", + "test/cpp/qps/perf_db_client.h", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.h", "test/cpp/qps/server.h", diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 13d77d571bf..675a2fb3b93 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -9781,6 +9781,7 @@ "test/cpp/qps/interarrival.h", "test/cpp/qps/perf_db.grpc.pb.h", "test/cpp/qps/perf_db.pb.h", + "test/cpp/qps/perf_db_client.h", "test/cpp/qps/qps_worker.h", "test/cpp/qps/qpstest.grpc.pb.h", "test/cpp/qps/qpstest.pb.h", @@ -9801,6 +9802,7 @@ "test/cpp/qps/histogram.h", "test/cpp/qps/interarrival.h", "test/cpp/qps/perf_db_client.cc", + "test/cpp/qps/perf_db_client.h", "test/cpp/qps/qps_worker.cc", "test/cpp/qps/qps_worker.h", "test/cpp/qps/report.cc", From fd1a20a667fc2cfdfe2afd8a8d54cb3f5438b5df Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Tue, 30 Jun 2015 16:38:32 -0700 Subject: [PATCH 31/63] Changes introduced for passing hashed user id instead of access token --- test/cpp/qps/perf_db.proto | 2 +- test/cpp/qps/perf_db_client.cc | 6 +++--- test/cpp/qps/perf_db_client.h | 2 +- test/cpp/qps/report.cc | 2 +- test/cpp/qps/report.h | 6 +++--- test/cpp/util/benchmark_config.cc | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/cpp/qps/perf_db.proto b/test/cpp/qps/perf_db.proto index 7a550f83c65..60e038406a8 100644 --- a/test/cpp/qps/perf_db.proto +++ b/test/cpp/qps/perf_db.proto @@ -57,7 +57,7 @@ message Metrics { // Request for storing a single user's data message SingleUserRecordRequest { - string access_token = 1; + string hashed_id = 1; string test_name = 2; string sys_info = 3; string tag = 4; diff --git a/test/cpp/qps/perf_db_client.cc b/test/cpp/qps/perf_db_client.cc index 5ec87e150c6..0996ea2b27c 100644 --- a/test/cpp/qps/perf_db_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -39,7 +39,7 @@ namespace testing { // sets the client and server config information void PerfDbClient::setConfigs(const ClientConfig& client_config, const ServerConfig& server_config) { - this->client_config_ = client_config; + client_config_ = client_config; this->server_config_ = server_config; } @@ -76,13 +76,13 @@ void PerfDbClient::setTimes(double server_system_time, double server_user_time, } // sends the data to the performance database server -bool PerfDbClient::sendData(std::string access_token, std::string test_name, +bool PerfDbClient::sendData(std::string hashed_id, std::string test_name, std::string sys_info, std::string tag) { // Data record request object SingleUserRecordRequest single_user_record_request; // setting access token, name of the test and the system information - single_user_record_request.set_access_token(access_token); + single_user_record_request.set_hashed_id(hashed_id); single_user_record_request.set_test_name(test_name); single_user_record_request.set_sys_info(sys_info); single_user_record_request.set_tag(tag); diff --git a/test/cpp/qps/perf_db_client.h b/test/cpp/qps/perf_db_client.h index a22426bced4..ce7a88bbff0 100644 --- a/test/cpp/qps/perf_db_client.h +++ b/test/cpp/qps/perf_db_client.h @@ -91,7 +91,7 @@ class PerfDbClient { double client_system_time, double client_user_time); // sends the data to the performance database server - bool sendData(std::string access_token, std::string test_name, + bool sendData(std::string hashed_id, std::string test_name, std::string sys_info, std::string tag); private: diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index da1dea62aec..ff01ec15014 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -172,7 +172,7 @@ void PerfDbReporter::ReportTimes(const ScenarioResult& result) { void PerfDbReporter::SendData() { // send data to performance database bool data_state = - perf_db_client_.sendData(access_token_, test_name_, sys_info_, tag_); + perf_db_client_.sendData(hashed_id_, test_name_, sys_info_, tag_); // check state of data sending if (data_state) { diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 88707bea387..aec3cbe80a3 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -107,11 +107,11 @@ class GprLogReporter : public Reporter { /** Reporter for performance database tool */ class PerfDbReporter : public Reporter { public: - PerfDbReporter(const string& name, const string& access_token, + PerfDbReporter(const string& name, const string& hashed_id, const string& test_name, const string& sys_info, const string& server_address, const string& tag) : Reporter(name), - access_token_(access_token), + hashed_id_(hashed_id), test_name_(test_name), sys_info_(sys_info), tag_(tag) { @@ -122,7 +122,7 @@ class PerfDbReporter : public Reporter { private: PerfDbClient perf_db_client_; - std::string access_token_; + std::string hashed_id_; std::string test_name_; std::string sys_info_; std::string tag_; diff --git a/test/cpp/util/benchmark_config.cc b/test/cpp/util/benchmark_config.cc index 030cb28c32f..91fbbf96771 100644 --- a/test/cpp/util/benchmark_config.cc +++ b/test/cpp/util/benchmark_config.cc @@ -39,7 +39,7 @@ DEFINE_bool(enable_log_reporter, true, DEFINE_bool(report_metrics_db, false, "True if metrics to be reported to performance database"); -DEFINE_string(access_token, "", "Authorizing JSON string for leaderboard"); +DEFINE_string(hashed_id, "", "Hash of the user id"); DEFINE_string(test_name, "", "Name of the test being executed"); @@ -71,7 +71,7 @@ static std::shared_ptr InitBenchmarkReporters() { } if(FLAGS_report_metrics_db) { composite_reporter->add( - std::unique_ptr(new PerfDbReporter("PerfDbReporter", FLAGS_access_token, FLAGS_test_name, + std::unique_ptr(new PerfDbReporter("PerfDbReporter", FLAGS_hashed_id, FLAGS_test_name, FLAGS_sys_info, FLAGS_server_address, FLAGS_tag))); } From a90c81ea61a736729598ae64c7c4a6ba2e2267ff Mon Sep 17 00:00:00 2001 From: Siddharth Rakesh Date: Tue, 30 Jun 2015 20:29:35 -0700 Subject: [PATCH 32/63] Removing this pointers --- test/cpp/qps/perf_db_client.cc | 50 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/test/cpp/qps/perf_db_client.cc b/test/cpp/qps/perf_db_client.cc index 0996ea2b27c..08d20f0b8d8 100644 --- a/test/cpp/qps/perf_db_client.cc +++ b/test/cpp/qps/perf_db_client.cc @@ -40,17 +40,17 @@ namespace testing { void PerfDbClient::setConfigs(const ClientConfig& client_config, const ServerConfig& server_config) { client_config_ = client_config; - this->server_config_ = server_config; + server_config_ = server_config; } // sets the QPS void PerfDbClient::setQps(double qps) { - this->qps_ = qps; + qps_ = qps; } // sets the QPS per core void PerfDbClient::setQpsPerCore(double qps_per_core) { - this->qps_per_core_ = qps_per_core; + qps_per_core_ = qps_per_core; } // sets the 50th, 90th, 95th, 99th and 99.9th percentile latency @@ -59,20 +59,20 @@ void PerfDbClient::setLatencies(double perc_lat_50, double perc_lat_95, double perc_lat_99, double perc_lat_99_point_9) { - this->perc_lat_50_ = perc_lat_50; - this->perc_lat_90_ = perc_lat_90; - this->perc_lat_95_ = perc_lat_95; - this->perc_lat_99_ = perc_lat_99; - this->perc_lat_99_point_9_ = perc_lat_99_point_9; + perc_lat_50_ = perc_lat_50; + perc_lat_90_ = perc_lat_90; + perc_lat_95_ = perc_lat_95; + perc_lat_99_ = perc_lat_99; + perc_lat_99_point_9_ = perc_lat_99_point_9; } // sets the server and client, user and system times void PerfDbClient::setTimes(double server_system_time, double server_user_time, double client_system_time, double client_user_time) { - this->server_system_time_ = server_system_time; - this->server_user_time_ = server_user_time; - this->client_system_time_ = client_system_time; - this->client_user_time_ = client_user_time; + server_system_time_ = server_system_time; + server_user_time_ = server_user_time; + client_system_time_ = client_system_time; + client_user_time_ = client_user_time; } // sends the data to the performance database server @@ -88,44 +88,44 @@ bool PerfDbClient::sendData(std::string hashed_id, std::string test_name, single_user_record_request.set_tag(tag); // setting configs - *(single_user_record_request.mutable_client_config()) = this->client_config_; - *(single_user_record_request.mutable_server_config()) = this->server_config_; + *(single_user_record_request.mutable_client_config()) = client_config_; + *(single_user_record_request.mutable_server_config()) = server_config_; Metrics* metrics = single_user_record_request.mutable_metrics(); // setting metrcs in data record request if (qps_ != DBL_MIN) { - metrics->set_qps(this->qps_); + metrics->set_qps(qps_); } if (qps_per_core_ != DBL_MIN) { - metrics->set_qps_per_core(this->qps_per_core_); + metrics->set_qps_per_core(qps_per_core_); } if (perc_lat_50_ != DBL_MIN) { - metrics->set_perc_lat_50(this->perc_lat_50_); + metrics->set_perc_lat_50(perc_lat_50_); } if (perc_lat_90_ != DBL_MIN) { - metrics->set_perc_lat_90(this->perc_lat_90_); + metrics->set_perc_lat_90(perc_lat_90_); } if (perc_lat_95_ != DBL_MIN) { - metrics->set_perc_lat_95(this->perc_lat_95_); + metrics->set_perc_lat_95(perc_lat_95_); } if (perc_lat_99_ != DBL_MIN) { - metrics->set_perc_lat_99(this->perc_lat_99_); + metrics->set_perc_lat_99(perc_lat_99_); } if (perc_lat_99_point_9_ != DBL_MIN) { - metrics->set_perc_lat_99_point_9(this->perc_lat_99_point_9_); + metrics->set_perc_lat_99_point_9(perc_lat_99_point_9_); } if (server_system_time_ != DBL_MIN) { - metrics->set_server_system_time(this->server_system_time_); + metrics->set_server_system_time(server_system_time_); } if (server_user_time_ != DBL_MIN) { - metrics->set_server_user_time(this->server_user_time_); + metrics->set_server_user_time(server_user_time_); } if (client_system_time_ != DBL_MIN) { - metrics->set_client_system_time(this->client_system_time_); + metrics->set_client_system_time(client_system_time_); } if (client_user_time_ != DBL_MIN) { - metrics->set_client_user_time(this->client_user_time_); + metrics->set_client_user_time(client_user_time_); } SingleUserRecordReply single_user_record_reply; From 560619958524aac022b54eb9dec51071684651ed Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 1 Jul 2015 13:39:48 -0700 Subject: [PATCH 33/63] Delete spare contexts lying around at the end of the test to avoid showing up as memory leaks Also reduce the rate of the openloop test - this may be an issue with sanitizers particularly because of issue #2278 --- test/cpp/qps/client_async.cc | 9 +++++++++ test/cpp/qps/qps_test_openloop.cc | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 8c8d927d15f..7c89a35ee7e 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -199,6 +199,15 @@ class AsyncClient : public Client { delete ClientRpcContext::detag(got_tag); } } + // Now clear out all the pre-allocated idle contexts + for (int ch = 0; ch < channel_count_; ch++) { + if (!contexts_[ch].empty()) { + // Get an idle context from the front of the list + auto* ctx = *(contexts_[ch].begin()); + contexts_[ch].pop_front(); + delete ctx; + } + } } bool ThreadFunc(Histogram* histogram, diff --git a/test/cpp/qps/qps_test_openloop.cc b/test/cpp/qps/qps_test_openloop.cc index 52873b2987e..96a9b4504c9 100644 --- a/test/cpp/qps/qps_test_openloop.cc +++ b/test/cpp/qps/qps_test_openloop.cc @@ -60,7 +60,7 @@ static void RunQPS() { client_config.set_rpc_type(UNARY); client_config.set_load_type(POISSON); client_config.mutable_load_params()-> - mutable_poisson()->set_offered_load(10000.0); + mutable_poisson()->set_offered_load(1000.0); ServerConfig server_config; server_config.set_server_type(ASYNC_SERVER); From 13735d5d7e8b9413ec6ba54af4eeb0e88cd0302f Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 1 Jul 2015 14:00:08 -0700 Subject: [PATCH 34/63] if->while --- test/cpp/qps/client_async.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 7c89a35ee7e..e1e44f9ac0d 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -201,7 +201,7 @@ class AsyncClient : public Client { } // Now clear out all the pre-allocated idle contexts for (int ch = 0; ch < channel_count_; ch++) { - if (!contexts_[ch].empty()) { + while (!contexts_[ch].empty()) { // Get an idle context from the front of the list auto* ctx = *(contexts_[ch].begin()); contexts_[ch].pop_front(); From 4bc49051ba823cfc9ddb342e08b1ab7638fa6396 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Wed, 1 Jul 2015 14:18:04 -0700 Subject: [PATCH 35/63] Skylark macro to generate an objc_library from protos Still needed: - Use compiled protoc. - Work with multiple proto sources. - Work with proto sources in nested dirs. --- grpc.bzl | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 grpc.bzl diff --git a/grpc.bzl b/grpc.bzl new file mode 100644 index 00000000000..9865bc734d3 --- /dev/null +++ b/grpc.bzl @@ -0,0 +1,68 @@ + +def _lower_underscore_to_upper_camel(str): + humps = [] + for hump in str.split('_'): + humps += [hump[0].upper() + hump[1:]] + return "".join(humps) + +def objc_grpc_library(name, srcs, visibility=None): + basename = srcs[0].split('/')[-1] + filename = basename[:-6] # remove .proto suffix + filename = _lower_underscore_to_upper_camel(filename) + + protoc_command = "protoc -I . " + srcs_params = "" + for src in srcs: + srcs_params += " $(location %s)" % (src) + + # Messages + protoc_messages_flags = "--objc_out=$(GENDIR)" + native.genrule( + name = name + "_mesages_codegen", + srcs = srcs, + outs = [ + filename + ".pbobjc.h", + filename + ".pbobjc.m", + ], + cmd = protoc_command + protoc_messages_flags + srcs_params, + ) + native.objc_library( + name = name + "_messages", + hdrs = [ + ":" + filename + ".pbobjc.h", + ], + includes = ["."], + non_arc_srcs = [ + ":" + filename + ".pbobjc.m", + ], + deps = [ + "//external:protobuf_objc", + ], + ) + + # Services + protoc_services_flags = "--grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(location //external:grpc_protoc_plugin_objc)" + native.genrule( + name = name + "_codegen", + srcs = srcs + ["//external:grpc_protoc_plugin_objc"], + outs = [ + filename + ".pbrpc.h", + filename + ".pbrpc.m", + ], + cmd = protoc_command + protoc_services_flags + srcs_params, + ) + native.objc_library( + name = name, + hdrs = [ + ":" + filename + ".pbrpc.h", + ], + includes = ["."], + srcs = [ + ":" + filename + ".pbrpc.m", + ], + deps = [ + ":" + name + "_messages", + "//external:proto_objc_rpc", + ], + visibility = visibility, + ) From 2daa88cfb464c08505d8f3579191b134f3417deb Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 1 Jul 2015 14:46:42 -0700 Subject: [PATCH 36/63] Fix a TSAN reported race close() could race with epoll_ctl(); pretend to be polling while adding to the epoll set to prevent this --- src/core/iomgr/fd_posix.c | 4 +-- .../iomgr/pollset_multipoller_with_epoll.c | 26 ++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c index bcc11f91b70..1297145d1a7 100644 --- a/src/core/iomgr/fd_posix.c +++ b/src/core/iomgr/fd_posix.c @@ -371,13 +371,13 @@ gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, return 0; } /* if there is nobody polling for read, but we need to, then start doing so */ - if (!fd->read_watcher && gpr_atm_acq_load(&fd->readst) > READY) { + if (read_mask && !fd->read_watcher && gpr_atm_acq_load(&fd->readst) > READY) { fd->read_watcher = watcher; mask |= read_mask; } /* if there is nobody polling for write, but we need to, then start doing so */ - if (!fd->write_watcher && gpr_atm_acq_load(&fd->writest) > READY) { + if (write_mask && !fd->write_watcher && gpr_atm_acq_load(&fd->writest) > READY) { fd->write_watcher = watcher; mask |= write_mask; } diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index dcf08d379cb..1900bbf9e14 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -54,17 +54,25 @@ static void multipoll_with_epoll_pollset_add_fd(grpc_pollset *pollset, pollset_hdr *h = pollset->data.ptr; struct epoll_event ev; int err; - - ev.events = EPOLLIN | EPOLLOUT | EPOLLET; - ev.data.ptr = fd; - err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); - if (err < 0) { - /* FDs may be added to a pollset multiple times, so EEXIST is normal. */ - if (errno != EEXIST) { - gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd, - strerror(errno)); + grpc_fd_watcher watcher; + + /* We pretend to be polling whilst adding an fd to keep the fd from being + closed during the add. This may result in a spurious wakeup being assigned + to this pollset whilst adding, but that should be benign. */ + GPR_ASSERT(grpc_fd_begin_poll(fd, pollset, 0, 0, &watcher) == 0); + if (watcher.fd != NULL) { + ev.events = EPOLLIN | EPOLLOUT | EPOLLET; + ev.data.ptr = fd; + err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev); + if (err < 0) { + /* FDs may be added to a pollset multiple times, so EEXIST is normal. */ + if (errno != EEXIST) { + gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd, + strerror(errno)); + } } } + grpc_fd_end_poll(&watcher, 0, 0); } static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset, From 27df2cf69caf015ffdd1df5945f9fc3faa94906f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 1 Jul 2015 15:11:29 -0700 Subject: [PATCH 37/63] Pluck some low hanging concurrency fruit Make the shutdown flag on servers be per thread to save contention on the lock that must guard it. --- test/cpp/qps/server_async.cc | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index 210aef4fd6b..f5251e961b9 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -64,7 +64,7 @@ namespace testing { class AsyncQpsServerTest : public Server { public: - AsyncQpsServerTest(const ServerConfig &config, int port) : shutdown_(false) { + AsyncQpsServerTest(const ServerConfig &config, int port) { char *server_address = NULL; gpr_join_host_port(&server_address, "::", port); @@ -96,6 +96,9 @@ class AsyncQpsServerTest : public Server { request_streaming, ProcessRPC)); } } + for (int i = 0; i < config.threads(); i++) { + shutdown_state_.emplace_back(new PerThreadShutdownState()); + } for (int i = 0; i < config.threads(); i++) { threads_.push_back(std::thread([=]() { // Wait until work is available or we are shutting down @@ -105,11 +108,9 @@ class AsyncQpsServerTest : public Server { ServerRpcContext *ctx = detag(got_tag); // The tag is a pointer to an RPC context to invoke bool still_going = ctx->RunNextState(ok); - std::unique_lock g(shutdown_mutex_); - if (!shutdown_) { + if (!shutdown_state_[i]->shutdown()) { // this RPC context is done, so refresh it if (!still_going) { - g.unlock(); ctx->Reset(); } } else { @@ -122,9 +123,8 @@ class AsyncQpsServerTest : public Server { } ~AsyncQpsServerTest() { server_->Shutdown(); - { - std::lock_guard g(shutdown_mutex_); - shutdown_ = true; + for (auto ss = shutdown_state_.begin(); ss != shutdown_state_.end(); ++ss) { + (*ss)->set_shutdown(); } for (auto thr = threads_.begin(); thr != threads_.end(); thr++) { thr->join(); @@ -316,8 +316,25 @@ class AsyncQpsServerTest : public Server { TestService::AsyncService async_service_; std::forward_list contexts_; - std::mutex shutdown_mutex_; - bool shutdown_; + class PerThreadShutdownState { + public: + PerThreadShutdownState() : shutdown_(false) {} + + bool shutdown() const { + std::lock_guard lock(mutex_); + return shutdown_; + } + + void set_shutdown() { + std::lock_guard lock(mutex_); + shutdown_ = true; + } + + private: + mutable std::mutex mutex_; + bool shutdown_; + }; + std::vector> shutdown_state_; }; std::unique_ptr CreateAsyncServer(const ServerConfig &config, From fc5fb879a6a05826f9b9078d172e2d59a1e8be43 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Wed, 1 Jul 2015 15:29:30 -0700 Subject: [PATCH 38/63] Support sources in nested dirs --- grpc.bzl | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/grpc.bzl b/grpc.bzl index 9865bc734d3..2ad3b03b576 100644 --- a/grpc.bzl +++ b/grpc.bzl @@ -6,7 +6,9 @@ def _lower_underscore_to_upper_camel(str): return "".join(humps) def objc_grpc_library(name, srcs, visibility=None): - basename = srcs[0].split('/')[-1] + src_path_elements = srcs[0].split('/') + src_dir = '/'.join(src_path_elements[:-1]) + basename = src_path_elements[-1] filename = basename[:-6] # remove .proto suffix filename = _lower_underscore_to_upper_camel(filename) @@ -17,24 +19,19 @@ def objc_grpc_library(name, srcs, visibility=None): # Messages protoc_messages_flags = "--objc_out=$(GENDIR)" + message_header = src_dir + '/' + filename + ".pbobjc.h" + message_implementation = src_dir + '/' + filename + ".pbobjc.m" native.genrule( name = name + "_mesages_codegen", srcs = srcs, - outs = [ - filename + ".pbobjc.h", - filename + ".pbobjc.m", - ], + outs = [message_header, message_implementation], cmd = protoc_command + protoc_messages_flags + srcs_params, ) native.objc_library( name = name + "_messages", - hdrs = [ - ":" + filename + ".pbobjc.h", - ], + hdrs = [message_header], includes = ["."], - non_arc_srcs = [ - ":" + filename + ".pbobjc.m", - ], + non_arc_srcs = [message_implementation], deps = [ "//external:protobuf_objc", ], @@ -42,24 +39,19 @@ def objc_grpc_library(name, srcs, visibility=None): # Services protoc_services_flags = "--grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(location //external:grpc_protoc_plugin_objc)" + service_header = src_dir + '/' + filename + ".pbrpc.h" + service_implementation = src_dir + '/' + filename + ".pbrpc.m" native.genrule( name = name + "_codegen", srcs = srcs + ["//external:grpc_protoc_plugin_objc"], - outs = [ - filename + ".pbrpc.h", - filename + ".pbrpc.m", - ], + outs = [service_header, service_implementation], cmd = protoc_command + protoc_services_flags + srcs_params, ) native.objc_library( name = name, - hdrs = [ - ":" + filename + ".pbrpc.h", - ], + hdrs = [service_header], includes = ["."], - srcs = [ - ":" + filename + ".pbrpc.m", - ], + srcs = [service_implementation], deps = [ ":" + name + "_messages", "//external:proto_objc_rpc", From 39fa7db46b22850882d9e44c2afbf9204532957c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 1 Jul 2015 15:40:21 -0700 Subject: [PATCH 39/63] Remove errant assert --- src/core/transport/chttp2/parsing.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/core/transport/chttp2/parsing.c b/src/core/transport/chttp2/parsing.c index 8f682e9017c..4664a0895c2 100644 --- a/src/core/transport/chttp2/parsing.c +++ b/src/core/transport/chttp2/parsing.c @@ -109,9 +109,6 @@ void grpc_chttp2_publish_reads( transport_parsing->incoming_stream_id; } - /* TODO(ctiller): re-implement */ - GPR_ASSERT(transport_parsing->initial_window_update == 0); - /* copy parsing qbuf to global qbuf */ gpr_slice_buffer_move_into(&transport_parsing->qbuf, &transport_global->qbuf); From 0e640a8cb22634b51a2e7e0d193449341ac92663 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 1 Jul 2015 15:41:52 -0700 Subject: [PATCH 40/63] Properly reset counter --- src/core/transport/chttp2_transport.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 94659a6bdf9..c7e9b3e13fa 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -908,6 +908,7 @@ static void recv_data(void *tp, gpr_slice *slices, size_t nslices, if (t->parsing.initial_window_update != 0) { grpc_chttp2_stream_map_for_each(&t->parsing_stream_map, update_global_window, t); + t->parsing.initial_window_update = 0; } /* handle higher level things */ grpc_chttp2_publish_reads(&t->global, &t->parsing); From df26af5f663e297d7c6d9af581352c2feceaa8c0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 1 Jul 2015 16:53:09 -0700 Subject: [PATCH 41/63] Fix refcount leak --- src/core/iomgr/fd_posix.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/iomgr/fd_posix.c b/src/core/iomgr/fd_posix.c index 1297145d1a7..632d2a46099 100644 --- a/src/core/iomgr/fd_posix.c +++ b/src/core/iomgr/fd_posix.c @@ -368,6 +368,7 @@ gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset, watcher->fd = NULL; watcher->pollset = NULL; gpr_mu_unlock(&fd->watcher_mu); + GRPC_FD_UNREF(fd, "poll"); return 0; } /* if there is nobody polling for read, but we need to, then start doing so */ From 164f633283259a3f1306e3546910c9fcaaa56bbb Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Wed, 1 Jul 2015 18:29:21 -0700 Subject: [PATCH 42/63] Extract filename transformations into functions --- grpc.bzl | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/grpc.bzl b/grpc.bzl index 2ad3b03b576..f31e188dd60 100644 --- a/grpc.bzl +++ b/grpc.bzl @@ -5,22 +5,28 @@ def _lower_underscore_to_upper_camel(str): humps += [hump[0].upper() + hump[1:]] return "".join(humps) -def objc_grpc_library(name, srcs, visibility=None): - src_path_elements = srcs[0].split('/') - src_dir = '/'.join(src_path_elements[:-1]) - basename = src_path_elements[-1] - filename = basename[:-6] # remove .proto suffix - filename = _lower_underscore_to_upper_camel(filename) +def _file_to_upper_camel(src): + elements = src.rpartition('/') + upper_camel = _lower_underscore_to_upper_camel(elements[-1]) + return "".join(elements[:-1] + [upper_camel]) + +def _file_with_extension(src, ext): + elements = src.rpartition('/') + basename = elements[-1].partition('.')[0] + return "".join(elements[:-1] + [basename, ext]) +def objc_grpc_library(name, srcs, visibility=None): protoc_command = "protoc -I . " srcs_params = "" for src in srcs: srcs_params += " $(location %s)" % (src) + src = _file_to_upper_camel(srcs[0]) + # Messages protoc_messages_flags = "--objc_out=$(GENDIR)" - message_header = src_dir + '/' + filename + ".pbobjc.h" - message_implementation = src_dir + '/' + filename + ".pbobjc.m" + message_header = _file_with_extension(src, ".pbobjc.h") + message_implementation = _file_with_extension(src, ".pbobjc.m") native.genrule( name = name + "_mesages_codegen", srcs = srcs, @@ -39,8 +45,8 @@ def objc_grpc_library(name, srcs, visibility=None): # Services protoc_services_flags = "--grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(location //external:grpc_protoc_plugin_objc)" - service_header = src_dir + '/' + filename + ".pbrpc.h" - service_implementation = src_dir + '/' + filename + ".pbrpc.m" + service_header = _file_with_extension(src, ".pbrpc.h") + service_implementation = _file_with_extension(src, ".pbrpc.m") native.genrule( name = name + "_codegen", srcs = srcs + ["//external:grpc_protoc_plugin_objc"], From d9435277fbb686da64ac831bc085f2670d5c2977 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Wed, 1 Jul 2015 18:43:54 -0700 Subject: [PATCH 43/63] Split messages and services macros --- grpc.bzl | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/grpc.bzl b/grpc.bzl index f31e188dd60..2e4528cb2c1 100644 --- a/grpc.bzl +++ b/grpc.bzl @@ -15,43 +15,49 @@ def _file_with_extension(src, ext): basename = elements[-1].partition('.')[0] return "".join(elements[:-1] + [basename, ext]) -def objc_grpc_library(name, srcs, visibility=None): +def _protoc_invocation(srcs, flags): protoc_command = "protoc -I . " srcs_params = "" for src in srcs: srcs_params += " $(location %s)" % (src) + return protoc_command + flags + srcs_params +def objc_proto_library(name, srcs, visibility=None): src = _file_to_upper_camel(srcs[0]) - # Messages - protoc_messages_flags = "--objc_out=$(GENDIR)" + protoc_flags = "--objc_out=$(GENDIR)" message_header = _file_with_extension(src, ".pbobjc.h") message_implementation = _file_with_extension(src, ".pbobjc.m") native.genrule( - name = name + "_mesages_codegen", + name = name + "_codegen", srcs = srcs, outs = [message_header, message_implementation], - cmd = protoc_command + protoc_messages_flags + srcs_params, + cmd = _protoc_invocation(srcs, protoc_flags), ) native.objc_library( - name = name + "_messages", + name = name, hdrs = [message_header], includes = ["."], non_arc_srcs = [message_implementation], deps = [ "//external:protobuf_objc", ], + visibility = visibility, ) - # Services - protoc_services_flags = "--grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(location //external:grpc_protoc_plugin_objc)" +def objc_grpc_library(name, srcs, visibility=None): + objc_proto_library(name + "_messages", srcs, visibility) + + src = _file_to_upper_camel(srcs[0]) + + protoc_flags = "--grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(location //external:grpc_protoc_plugin_objc)" service_header = _file_with_extension(src, ".pbrpc.h") service_implementation = _file_with_extension(src, ".pbrpc.m") native.genrule( name = name + "_codegen", srcs = srcs + ["//external:grpc_protoc_plugin_objc"], outs = [service_header, service_implementation], - cmd = protoc_command + protoc_services_flags + srcs_params, + cmd = _protoc_invocation(srcs, protoc_flags), ) native.objc_library( name = name, From 907fad9c4d03e081b670492a5b872628a0c7a8ac Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Thu, 2 Jul 2015 00:51:01 -0700 Subject: [PATCH 44/63] Support multiple proto sources --- grpc.bzl | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/grpc.bzl b/grpc.bzl index 2e4528cb2c1..0e94c4faf81 100644 --- a/grpc.bzl +++ b/grpc.bzl @@ -23,47 +23,53 @@ def _protoc_invocation(srcs, flags): return protoc_command + flags + srcs_params def objc_proto_library(name, srcs, visibility=None): - src = _file_to_upper_camel(srcs[0]) + h_files = [] + m_files = [] + for src in srcs: + src = _file_to_upper_camel(src) + h_files += [_file_with_extension(src, ".pbobjc.h")] + m_files += [_file_with_extension(src, ".pbobjc.m")] protoc_flags = "--objc_out=$(GENDIR)" - message_header = _file_with_extension(src, ".pbobjc.h") - message_implementation = _file_with_extension(src, ".pbobjc.m") + native.genrule( name = name + "_codegen", srcs = srcs, - outs = [message_header, message_implementation], + outs = h_files + m_files, cmd = _protoc_invocation(srcs, protoc_flags), ) native.objc_library( name = name, - hdrs = [message_header], + hdrs = h_files, includes = ["."], - non_arc_srcs = [message_implementation], - deps = [ - "//external:protobuf_objc", - ], + non_arc_srcs = m_files, + deps = ["//external:protobuf_objc"], visibility = visibility, ) -def objc_grpc_library(name, srcs, visibility=None): - objc_proto_library(name + "_messages", srcs, visibility) +def objc_grpc_library(name, services, other_messages, visibility=None): + objc_proto_library(name + "_messages", services + other_messages) - src = _file_to_upper_camel(srcs[0]) + h_files = [] + m_files = [] + for src in services: + src = _file_to_upper_camel(src) + h_files += [_file_with_extension(src, ".pbrpc.h")] + m_files += [_file_with_extension(src, ".pbrpc.m")] protoc_flags = "--grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(location //external:grpc_protoc_plugin_objc)" - service_header = _file_with_extension(src, ".pbrpc.h") - service_implementation = _file_with_extension(src, ".pbrpc.m") + native.genrule( name = name + "_codegen", - srcs = srcs + ["//external:grpc_protoc_plugin_objc"], - outs = [service_header, service_implementation], - cmd = _protoc_invocation(srcs, protoc_flags), + srcs = services + ["//external:grpc_protoc_plugin_objc"], + outs = h_files + m_files, + cmd = _protoc_invocation(services, protoc_flags), ) native.objc_library( name = name, - hdrs = [service_header], + hdrs = h_files, includes = ["."], - srcs = [service_implementation], + srcs = m_files, deps = [ ":" + name + "_messages", "//external:proto_objc_rpc", From 8637ececfd15c7325cf99a965a54387c7d994ad8 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Thu, 2 Jul 2015 01:01:53 -0700 Subject: [PATCH 45/63] Use a protoc compiled from sources --- grpc.bzl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/grpc.bzl b/grpc.bzl index 0e94c4faf81..5b5735e1bc4 100644 --- a/grpc.bzl +++ b/grpc.bzl @@ -16,7 +16,7 @@ def _file_with_extension(src, ext): return "".join(elements[:-1] + [basename, ext]) def _protoc_invocation(srcs, flags): - protoc_command = "protoc -I . " + protoc_command = "$(location //external:protoc) -I . " srcs_params = "" for src in srcs: srcs_params += " $(location %s)" % (src) @@ -34,7 +34,7 @@ def objc_proto_library(name, srcs, visibility=None): native.genrule( name = name + "_codegen", - srcs = srcs, + srcs = srcs + ["//external:protoc"], outs = h_files + m_files, cmd = _protoc_invocation(srcs, protoc_flags), ) @@ -61,7 +61,10 @@ def objc_grpc_library(name, services, other_messages, visibility=None): native.genrule( name = name + "_codegen", - srcs = services + ["//external:grpc_protoc_plugin_objc"], + srcs = services + [ + "//external:grpc_protoc_plugin_objc", + "//external:protoc", + ], outs = h_files + m_files, cmd = _protoc_invocation(services, protoc_flags), ) From 24b2f67b80320d1f215c6465ea0092b7b01caa04 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Thu, 2 Jul 2015 01:29:37 -0700 Subject: [PATCH 46/63] Add copyright notice and documentation to .bzl file. --- grpc.bzl | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/grpc.bzl b/grpc.bzl index 5b5735e1bc4..9f2693126af 100644 --- a/grpc.bzl +++ b/grpc.bzl @@ -1,3 +1,39 @@ +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +Bazel macros to declare gRPC libraries automatically generated from proto files. + +This file declares two macros: +- objc_proto_library +- objc_grpc_library +""" def _lower_underscore_to_upper_camel(str): humps = [] @@ -16,6 +52,9 @@ def _file_with_extension(src, ext): return "".join(elements[:-1] + [basename, ext]) def _protoc_invocation(srcs, flags): + """Returns a command line to invoke protoc from a genrule, on the given + sources, using the given flags. + """ protoc_command = "$(location //external:protoc) -I . " srcs_params = "" for src in srcs: @@ -23,6 +62,9 @@ def _protoc_invocation(srcs, flags): return protoc_command + flags + srcs_params def objc_proto_library(name, srcs, visibility=None): + """Declares an objc_library for the code generated by protoc from the given + proto sources. This generated code doesn't include proto services. + """ h_files = [] m_files = [] for src in srcs: @@ -48,6 +90,10 @@ def objc_proto_library(name, srcs, visibility=None): ) def objc_grpc_library(name, services, other_messages, visibility=None): + """Declares an objc_library for the code generated by gRPC and protoc from the + given proto sources (services and other_messages). The generated code doesn't + include proto services of the files passed as other_messages. + """ objc_proto_library(name + "_messages", services + other_messages) h_files = [] @@ -57,7 +103,8 @@ def objc_grpc_library(name, services, other_messages, visibility=None): h_files += [_file_with_extension(src, ".pbrpc.h")] m_files += [_file_with_extension(src, ".pbrpc.m")] - protoc_flags = "--grpc_out=$(GENDIR) --plugin=protoc-gen-grpc=$(location //external:grpc_protoc_plugin_objc)" + protoc_flags = ("--grpc_out=$(GENDIR) --plugin=" + + "protoc-gen-grpc=$(location //external:grpc_protoc_plugin_objc)") native.genrule( name = name + "_codegen", From 3beb240544bd7a7dc192c2c2fb343e924d5bece0 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Thu, 2 Jul 2015 01:30:33 -0700 Subject: [PATCH 47/63] Minor doc fix in BUILD and BUILD.template --- BUILD | 2 +- templates/BUILD.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILD b/BUILD index 8a827ff54e1..b98c0883d5c 100644 --- a/BUILD +++ b/BUILD @@ -1,5 +1,5 @@ # GRPC Bazel BUILD file. -# This currently builds C and C++ code. +# This currently builds C, C++ and Objective-C code. # This file has been automatically generated from a template file. # Please look at the templates directory instead. # This file can be regenerated from the template by running diff --git a/templates/BUILD.template b/templates/BUILD.template index dffdc1dddde..4e9d8c376a9 100644 --- a/templates/BUILD.template +++ b/templates/BUILD.template @@ -1,5 +1,5 @@ # GRPC Bazel BUILD file. -# This currently builds C and C++ code. +# This currently builds C, C++ and Objective-C code. # This file has been automatically generated from a template file. # Please look at the templates directory instead. # This file can be regenerated from the template by running From cae5bf586f122871111229ace6a86dcdb8214e45 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 2 Jul 2015 08:46:59 -0700 Subject: [PATCH 48/63] Structure repacking and sizing --- src/core/iomgr/alarm.h | 2 +- src/core/surface/call.c | 8 ++++---- src/core/surface/call.h | 2 +- src/core/transport/stream_op.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/core/iomgr/alarm.h b/src/core/iomgr/alarm.h index e5262e21998..c067a0b8a3e 100644 --- a/src/core/iomgr/alarm.h +++ b/src/core/iomgr/alarm.h @@ -41,9 +41,9 @@ typedef struct grpc_alarm { gpr_timespec deadline; gpr_uint32 heap_index; /* INVALID_HEAP_INDEX if not in heap */ + int triggered; struct grpc_alarm *next; struct grpc_alarm *prev; - int triggered; grpc_iomgr_cb_func cb; void *cb_arg; } grpc_alarm; diff --git a/src/core/surface/call.c b/src/core/surface/call.c index 181617fff80..f6aeed856ba 100644 --- a/src/core/surface/call.c +++ b/src/core/surface/call.c @@ -76,14 +76,14 @@ typedef struct { typedef struct { /* Overall status of the operation: starts OK, may degrade to non-OK */ - int success; - /* Completion function to call at the end of the operation */ - grpc_ioreq_completion_func on_complete; - void *user_data; + gpr_uint8 success; /* a bit mask of which request ops are needed (1u << opid) */ gpr_uint16 need_mask; /* a bit mask of which request ops are now completed */ gpr_uint16 complete_mask; + /* Completion function to call at the end of the operation */ + grpc_ioreq_completion_func on_complete; + void *user_data; } reqinfo_master; /* Status data for a request can come from several sources; this diff --git a/src/core/surface/call.h b/src/core/surface/call.h index fb3662b50d3..3b6f9c942eb 100644 --- a/src/core/surface/call.h +++ b/src/core/surface/call.h @@ -78,8 +78,8 @@ typedef union { typedef struct { grpc_ioreq_op op; - grpc_ioreq_data data; gpr_uint32 flags; /**< A copy of the write flags from grpc_op */ + grpc_ioreq_data data; } grpc_ioreq; typedef void (*grpc_ioreq_completion_func)(grpc_call *call, int success, diff --git a/src/core/transport/stream_op.h b/src/core/transport/stream_op.h index 842fc932b96..964d39d14fc 100644 --- a/src/core/transport/stream_op.h +++ b/src/core/transport/stream_op.h @@ -41,7 +41,7 @@ #include "src/core/transport/metadata.h" /* this many stream ops are inlined into a sopb before allocating */ -#define GRPC_SOPB_INLINE_ELEMENTS 16 +#define GRPC_SOPB_INLINE_ELEMENTS 4 /* Operations that can be performed on a stream. Used by grpc_stream_op. */ From 9506819c5013f6a2e8561fc66ebcf86dec174f96 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Thu, 2 Jul 2015 20:18:43 -0700 Subject: [PATCH 49/63] Fix build breakage of moving include/grpc/ into grpc/ --- gRPC.podspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gRPC.podspec b/gRPC.podspec index 1a4dfd4c9a6..138cf96fd04 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -494,9 +494,9 @@ Pod::Spec.new do |s| BAD_TIME="$DIR_TIME/time.h" GOOD_TIME="$DIR_TIME/grpc_time.h" grep -rl "$BAD_TIME" grpc src/core src/objective-c/GRPCClient | xargs sed -i '' -e s@$BAD_TIME@$GOOD_TIME@g - if [ -f "include/$BAD_TIME" ]; + if [ -f "$BAD_TIME" ]; then - mv -f "include/$BAD_TIME" "include/$GOOD_TIME" + mv -f "$BAD_TIME" "$GOOD_TIME" fi DIR_STRING="src/core/support" From 951cd18309958d7ad2992d43ac565e91089b5d8c Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Thu, 2 Jul 2015 20:48:50 -0700 Subject: [PATCH 50/63] Fix build breakage of moving include/grpc/ into grpc/ --- templates/gRPC.podspec.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/gRPC.podspec.template b/templates/gRPC.podspec.template index deea07cee35..495ea49c9c2 100644 --- a/templates/gRPC.podspec.template +++ b/templates/gRPC.podspec.template @@ -111,9 +111,9 @@ Pod::Spec.new do |s| BAD_TIME="$DIR_TIME/time.h" GOOD_TIME="$DIR_TIME/grpc_time.h" grep -rl "$BAD_TIME" grpc src/core src/objective-c/GRPCClient | xargs sed -i '' -e s@$BAD_TIME@$GOOD_TIME@g - if [ -f "include/$BAD_TIME" ]; + if [ -f "$BAD_TIME" ]; then - mv -f "include/$BAD_TIME" "include/$GOOD_TIME" + mv -f "$BAD_TIME" "$GOOD_TIME" fi DIR_STRING="src/core/support" From 469d4b6fde8084232cafa3972691d20b6cb359c7 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 3 Jul 2015 12:02:38 -0700 Subject: [PATCH 51/63] Rename GRPCMethodName->ProtoMethod --- src/objective-c/GRPCClient/GRPCCall.h | 4 ++-- src/objective-c/GRPCClient/GRPCCall.m | 4 ++-- .../GRPCClient/{GRPCMethodName.h => ProtoMethod.h} | 2 +- .../GRPCClient/{GRPCMethodName.m => ProtoMethod.m} | 4 ++-- .../private/GRPCMethodName+HTTP2Encoding.h | 4 ++-- .../private/GRPCMethodName+HTTP2Encoding.m | 2 +- src/objective-c/ProtoRPC/ProtoRPC.h | 2 +- src/objective-c/ProtoRPC/ProtoRPC.m | 4 ++-- src/objective-c/ProtoRPC/ProtoService.m | 4 ++-- src/objective-c/tests/GRPCClientTests.m | 14 +++++++------- src/objective-c/tests/LocalClearTextTests.m | 6 +++--- 11 files changed, 25 insertions(+), 25 deletions(-) rename src/objective-c/GRPCClient/{GRPCMethodName.h => ProtoMethod.h} (98%) rename src/objective-c/GRPCClient/{GRPCMethodName.m => ProtoMethod.m} (97%) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 33aae107475..ed59817952c 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -48,7 +48,7 @@ #import #import -@class GRPCMethodName; +@class ProtoMethod; // Key used in |NSError|'s |userInfo| dictionary to store the response metadata sent by the server. extern id const kGRPCStatusMetadataKey; @@ -90,7 +90,7 @@ extern id const kGRPCStatusMetadataKey; // the specific remote method called). // To finish a call right away, invoke cancel. - (instancetype)initWithHost:(NSString *)host - method:(GRPCMethodName *)method + method:(ProtoMethod *)method requestsWriter:(id)requestsWriter NS_DESIGNATED_INITIALIZER; // Finishes the request side of this call, notifies the server that the RPC diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 77eebeff761..08b28282941 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -36,7 +36,7 @@ #include #include -#import "GRPCMethodName.h" +#import "ProtoMethod.h" #import "private/GRPCChannel.h" #import "private/GRPCCompletionQueue.h" #import "private/GRPCDelegateWrapper.h" @@ -95,7 +95,7 @@ NSString * const kGRPCStatusMetadataKey = @"io.grpc.StatusMetadataKey"; // Designated initializer - (instancetype)initWithHost:(NSString *)host - method:(GRPCMethodName *)method + method:(ProtoMethod *)method requestsWriter:(id)requestWriter { if (!host || !method) { [NSException raise:NSInvalidArgumentException format:@"Neither host nor method can be nil."]; diff --git a/src/objective-c/GRPCClient/GRPCMethodName.h b/src/objective-c/GRPCClient/ProtoMethod.h similarity index 98% rename from src/objective-c/GRPCClient/GRPCMethodName.h rename to src/objective-c/GRPCClient/ProtoMethod.h index fe153dd478b..814c1564641 100644 --- a/src/objective-c/GRPCClient/GRPCMethodName.h +++ b/src/objective-c/GRPCClient/ProtoMethod.h @@ -39,7 +39,7 @@ // implement multiple interfaces. // TODO(jcanizales): Move to ProtoRPC package. // TODO(jcanizales): Rename interface -> service. -@interface GRPCMethodName : NSObject +@interface ProtoMethod : NSObject @property(nonatomic, readonly) NSString *package; @property(nonatomic, readonly) NSString *interface; @property(nonatomic, readonly) NSString *method; diff --git a/src/objective-c/GRPCClient/GRPCMethodName.m b/src/objective-c/GRPCClient/ProtoMethod.m similarity index 97% rename from src/objective-c/GRPCClient/GRPCMethodName.m rename to src/objective-c/GRPCClient/ProtoMethod.m index 96724073a5f..1f5f836b7d0 100644 --- a/src/objective-c/GRPCClient/GRPCMethodName.m +++ b/src/objective-c/GRPCClient/ProtoMethod.m @@ -31,9 +31,9 @@ * */ -#import "GRPCMethodName.h" +#import "ProtoMethod.h" -@implementation GRPCMethodName +@implementation ProtoMethod - (instancetype)initWithPackage:(NSString *)package interface:(NSString *)interface method:(NSString *)method { diff --git a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.h b/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.h index 81c80f2a492..ac1872879cd 100644 --- a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.h +++ b/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.h @@ -33,8 +33,8 @@ #import -#import "GRPCClient/GRPCMethodName.h" +#import "GRPCClient/ProtoMethod.h" -@interface GRPCMethodName (HTTP2Encoding) +@interface ProtoMethod (HTTP2Encoding) - (NSString *)HTTP2Path; @end diff --git a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m b/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m index 3ad757fb29a..f31d33408b0 100644 --- a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m +++ b/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m @@ -33,7 +33,7 @@ #import "GRPCMethodName+HTTP2Encoding.h" -@implementation GRPCMethodName (HTTP2Encoding) +@implementation ProtoMethod (HTTP2Encoding) - (NSString *)HTTP2Path { if (self.package) { return [NSString stringWithFormat:@"/%@.%@/%@", self.package, self.interface, self.method]; diff --git a/src/objective-c/ProtoRPC/ProtoRPC.h b/src/objective-c/ProtoRPC/ProtoRPC.h index a3833106195..c5ab345c9cf 100644 --- a/src/objective-c/ProtoRPC/ProtoRPC.h +++ b/src/objective-c/ProtoRPC/ProtoRPC.h @@ -37,7 +37,7 @@ @interface ProtoRPC : GRPCCall - (instancetype)initWithHost:(NSString *)host - method:(GRPCMethodName *)method + method:(ProtoMethod *)method requestsWriter:(id)requestsWriter responseClass:(Class)responseClass responsesWriteable:(id)responsesWriteable NS_DESIGNATED_INITIALIZER; diff --git a/src/objective-c/ProtoRPC/ProtoRPC.m b/src/objective-c/ProtoRPC/ProtoRPC.m index 4da646d7b45..ec38077b714 100644 --- a/src/objective-c/ProtoRPC/ProtoRPC.m +++ b/src/objective-c/ProtoRPC/ProtoRPC.m @@ -43,7 +43,7 @@ } - (instancetype)initWithHost:(NSString *)host - method:(GRPCMethodName *)method + method:(ProtoMethod *)method requestsWriter:(id)requestsWriter { return [self initWithHost:host method:method @@ -54,7 +54,7 @@ // Designated initializer - (instancetype)initWithHost:(NSString *)host - method:(GRPCMethodName *)method + method:(ProtoMethod *)method requestsWriter:(id)requestsWriter responseClass:(Class)responseClass responsesWriteable:(id)responsesWriteable { diff --git a/src/objective-c/ProtoRPC/ProtoService.m b/src/objective-c/ProtoRPC/ProtoService.m index 47bdb5dc6e8..e8939389332 100644 --- a/src/objective-c/ProtoRPC/ProtoService.m +++ b/src/objective-c/ProtoRPC/ProtoService.m @@ -33,7 +33,7 @@ #import "ProtoService.h" -#import +#import #import #import @@ -69,7 +69,7 @@ requestsWriter:(id)requestsWriter responseClass:(Class)responseClass responsesWriteable:(id)responsesWriteable { - GRPCMethodName *methodName = [[GRPCMethodName alloc] initWithPackage:_packageName + ProtoMethod *methodName = [[ProtoMethod alloc] initWithPackage:_packageName interface:_serviceName method:method]; return [[ProtoRPC alloc] initWithHost:_host diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index e421127ea1e..e3b37b050e4 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -35,7 +35,7 @@ #import #import -#import +#import #import #import #import @@ -47,9 +47,9 @@ static NSString * const kHostAddress = @"grpc-test.sandbox.google.com"; static NSString * const kPackage = @"grpc.testing"; static NSString * const kService = @"TestService"; -static GRPCMethodName *kInexistentMethod; -static GRPCMethodName *kEmptyCallMethod; -static GRPCMethodName *kUnaryCallMethod; +static ProtoMethod *kInexistentMethod; +static ProtoMethod *kEmptyCallMethod; +static ProtoMethod *kUnaryCallMethod; @interface GRPCClientTests : XCTestCase @end @@ -58,13 +58,13 @@ static GRPCMethodName *kUnaryCallMethod; - (void)setUp { // This method isn't implemented by the remote server. - kInexistentMethod = [[GRPCMethodName alloc] initWithPackage:kPackage + kInexistentMethod = [[ProtoMethod alloc] initWithPackage:kPackage interface:kService method:@"Inexistent"]; - kEmptyCallMethod = [[GRPCMethodName alloc] initWithPackage:kPackage + kEmptyCallMethod = [[ProtoMethod alloc] initWithPackage:kPackage interface:kService method:@"EmptyCall"]; - kUnaryCallMethod = [[GRPCMethodName alloc] initWithPackage:kPackage + kUnaryCallMethod = [[ProtoMethod alloc] initWithPackage:kPackage interface:kService method:@"UnaryCall"]; } diff --git a/src/objective-c/tests/LocalClearTextTests.m b/src/objective-c/tests/LocalClearTextTests.m index 05cc10410a1..ace100847fe 100644 --- a/src/objective-c/tests/LocalClearTextTests.m +++ b/src/objective-c/tests/LocalClearTextTests.m @@ -35,7 +35,7 @@ #import #import -#import +#import #import #import #import @@ -87,7 +87,7 @@ static NSString * const kService = @"RouteGuide"; __weak XCTestExpectation *response = [self expectationWithDescription:@"Empty response received."]; __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; - GRPCMethodName *method = [[GRPCMethodName alloc] initWithPackage:kPackage + ProtoMethod *method = [[ProtoMethod alloc] initWithPackage:kPackage interface:kService method:@"RecordRoute"]; @@ -115,7 +115,7 @@ static NSString * const kService = @"RouteGuide"; __weak XCTestExpectation *response = [self expectationWithDescription:@"Response received."]; __weak XCTestExpectation *completion = [self expectationWithDescription:@"RPC completed."]; - GRPCMethodName *method = [[GRPCMethodName alloc] initWithPackage:kPackage + ProtoMethod *method = [[ProtoMethod alloc] initWithPackage:kPackage interface:kService method:@"GetFeature"]; From 1ac8f9a26aab6fc6cfff967073e81ec3c84a07c2 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Fri, 3 Jul 2015 12:05:04 -0700 Subject: [PATCH 52/63] Rename interface->service --- src/objective-c/GRPCClient/ProtoMethod.h | 3 +-- src/objective-c/GRPCClient/ProtoMethod.m | 2 +- .../GRPCClient/private/GRPCMethodName+HTTP2Encoding.m | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/objective-c/GRPCClient/ProtoMethod.h b/src/objective-c/GRPCClient/ProtoMethod.h index 814c1564641..86d582a03e5 100644 --- a/src/objective-c/GRPCClient/ProtoMethod.h +++ b/src/objective-c/GRPCClient/ProtoMethod.h @@ -38,10 +38,9 @@ // A fully-qualified gRPC method name. Full qualification is needed because a gRPC endpoint can // implement multiple interfaces. // TODO(jcanizales): Move to ProtoRPC package. -// TODO(jcanizales): Rename interface -> service. @interface ProtoMethod : NSObject @property(nonatomic, readonly) NSString *package; -@property(nonatomic, readonly) NSString *interface; +@property(nonatomic, readonly) NSString *service; @property(nonatomic, readonly) NSString *method; - (instancetype)initWithPackage:(NSString *)package interface:(NSString *)interface diff --git a/src/objective-c/GRPCClient/ProtoMethod.m b/src/objective-c/GRPCClient/ProtoMethod.m index 1f5f836b7d0..4c78b4f61c7 100644 --- a/src/objective-c/GRPCClient/ProtoMethod.m +++ b/src/objective-c/GRPCClient/ProtoMethod.m @@ -39,7 +39,7 @@ method:(NSString *)method { if ((self = [super init])) { _package = [package copy]; - _interface = [interface copy]; + _service = [interface copy]; _method = [method copy]; } return self; diff --git a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m b/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m index f31d33408b0..cb4efc82cfd 100644 --- a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m +++ b/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m @@ -36,9 +36,9 @@ @implementation ProtoMethod (HTTP2Encoding) - (NSString *)HTTP2Path { if (self.package) { - return [NSString stringWithFormat:@"/%@.%@/%@", self.package, self.interface, self.method]; + return [NSString stringWithFormat:@"/%@.%@/%@", self.package, self.service, self.method]; } else { - return [NSString stringWithFormat:@"/%@/%@", self.interface, self.method]; + return [NSString stringWithFormat:@"/%@/%@", self.service, self.method]; } } @end From be808e36f6dc9fd73b031520baf3e94c81307230 Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Sat, 4 Jul 2015 14:37:58 -0700 Subject: [PATCH 53/63] Move ProtoMethod to ProtoRPC package --- src/objective-c/GRPCClient/GRPCCall.h | 4 +- src/objective-c/GRPCClient/GRPCCall.m | 10 ++--- .../private/GRPCMethodName+HTTP2Encoding.h | 40 ----------------- .../private/GRPCMethodName+HTTP2Encoding.m | 44 ------------------- .../GRPCClient/private/GRPCWrappedCall.h | 2 +- .../GRPCClient/private/GRPCWrappedCall.m | 8 ++-- .../{GRPCClient => ProtoRPC}/ProtoMethod.h | 12 ++--- .../{GRPCClient => ProtoRPC}/ProtoMethod.m | 12 ++++- src/objective-c/ProtoRPC/ProtoRPC.h | 2 + src/objective-c/ProtoRPC/ProtoRPC.m | 15 ++++--- src/objective-c/ProtoRPC/ProtoService.m | 6 +-- src/objective-c/tests/GRPCClientTests.m | 22 +++++----- src/objective-c/tests/LocalClearTextTests.m | 14 +++--- 13 files changed, 57 insertions(+), 134 deletions(-) delete mode 100644 src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.h delete mode 100644 src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m rename src/objective-c/{GRPCClient => ProtoRPC}/ProtoMethod.h (86%) rename src/objective-c/{GRPCClient => ProtoRPC}/ProtoMethod.m (86%) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index ed59817952c..cba53fa2f67 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -48,8 +48,6 @@ #import #import -@class ProtoMethod; - // Key used in |NSError|'s |userInfo| dictionary to store the response metadata sent by the server. extern id const kGRPCStatusMetadataKey; @@ -90,7 +88,7 @@ extern id const kGRPCStatusMetadataKey; // the specific remote method called). // To finish a call right away, invoke cancel. - (instancetype)initWithHost:(NSString *)host - method:(ProtoMethod *)method + path:(NSString *)path requestsWriter:(id)requestsWriter NS_DESIGNATED_INITIALIZER; // Finishes the request side of this call, notifies the server that the RPC diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 08b28282941..4ac4e4d37f5 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -36,11 +36,9 @@ #include #include -#import "ProtoMethod.h" #import "private/GRPCChannel.h" #import "private/GRPCCompletionQueue.h" #import "private/GRPCDelegateWrapper.h" -#import "private/GRPCMethodName+HTTP2Encoding.h" #import "private/GRPCWrappedCall.h" #import "private/NSData+GRPC.h" #import "private/NSDictionary+GRPC.h" @@ -90,14 +88,14 @@ NSString * const kGRPCStatusMetadataKey = @"io.grpc.StatusMetadataKey"; @synthesize state = _state; - (instancetype)init { - return [self initWithHost:nil method:nil requestsWriter:nil]; + return [self initWithHost:nil path:nil requestsWriter:nil]; } // Designated initializer - (instancetype)initWithHost:(NSString *)host - method:(ProtoMethod *)method + path:(NSString *)path requestsWriter:(id)requestWriter { - if (!host || !method) { + if (!host || !path) { [NSException raise:NSInvalidArgumentException format:@"Neither host nor method can be nil."]; } if (requestWriter.state != GRXWriterStateNotStarted) { @@ -114,7 +112,7 @@ NSString * const kGRPCStatusMetadataKey = @"io.grpc.StatusMetadataKey"; _channel = [GRPCChannel channelToHost:host]; _wrappedCall = [[GRPCWrappedCall alloc] initWithChannel:_channel - method:method.HTTP2Path + path:path host:host]; // Serial queue to invoke the non-reentrant methods of the grpc_call object. diff --git a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.h b/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.h deleted file mode 100644 index ac1872879cd..00000000000 --- a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#import - -#import "GRPCClient/ProtoMethod.h" - -@interface ProtoMethod (HTTP2Encoding) -- (NSString *)HTTP2Path; -@end diff --git a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m b/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m deleted file mode 100644 index cb4efc82cfd..00000000000 --- a/src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.m +++ /dev/null @@ -1,44 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#import "GRPCMethodName+HTTP2Encoding.h" - -@implementation ProtoMethod (HTTP2Encoding) -- (NSString *)HTTP2Path { - if (self.package) { - return [NSString stringWithFormat:@"/%@.%@/%@", self.package, self.service, self.method]; - } else { - return [NSString stringWithFormat:@"/%@/%@", self.service, self.method]; - } -} -@end diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h index c08aefc6a8f..18f8bb55310 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.h +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.h @@ -84,7 +84,7 @@ @interface GRPCWrappedCall : NSObject - (instancetype)initWithChannel:(GRPCChannel *)channel - method:(NSString *)method + path:(NSString *)path host:(NSString *)host NS_DESIGNATED_INITIALIZER; - (void)startBatchWithOperations:(NSArray *)ops errorHandler:(void(^)())errorHandler; diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index d94b25091e6..45f10f5d63d 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -225,13 +225,13 @@ } - (instancetype)init { - return [self initWithChannel:nil method:nil host:nil]; + return [self initWithChannel:nil path:nil host:nil]; } - (instancetype)initWithChannel:(GRPCChannel *)channel - method:(NSString *)method + path:(NSString *)path host:(NSString *)host { - if (!channel || !method || !host) { + if (!channel || !path || !host) { [NSException raise:NSInvalidArgumentException format:@"channel, method, and host cannot be nil."]; } @@ -247,7 +247,7 @@ return nil; } _call = grpc_channel_create_call(channel.unmanagedChannel, _queue.unmanagedQueue, - method.UTF8String, host.UTF8String, gpr_inf_future); + path.UTF8String, host.UTF8String, gpr_inf_future); if (_call == NULL) { return nil; } diff --git a/src/objective-c/GRPCClient/ProtoMethod.h b/src/objective-c/ProtoRPC/ProtoMethod.h similarity index 86% rename from src/objective-c/GRPCClient/ProtoMethod.h rename to src/objective-c/ProtoRPC/ProtoMethod.h index 86d582a03e5..9ecb55a48fc 100644 --- a/src/objective-c/GRPCClient/ProtoMethod.h +++ b/src/objective-c/ProtoRPC/ProtoMethod.h @@ -33,16 +33,16 @@ #import -// See the README file for an introduction to this library. - -// A fully-qualified gRPC method name. Full qualification is needed because a gRPC endpoint can -// implement multiple interfaces. -// TODO(jcanizales): Move to ProtoRPC package. +// A fully-qualified proto service method name. Full qualification is needed because a gRPC endpoint +// can implement multiple services. @interface ProtoMethod : NSObject @property(nonatomic, readonly) NSString *package; @property(nonatomic, readonly) NSString *service; @property(nonatomic, readonly) NSString *method; + +@property(nonatomic, readonly) NSString *HTTP2Path; + - (instancetype)initWithPackage:(NSString *)package - interface:(NSString *)interface + service:(NSString *)service method:(NSString *)method; @end diff --git a/src/objective-c/GRPCClient/ProtoMethod.m b/src/objective-c/ProtoRPC/ProtoMethod.m similarity index 86% rename from src/objective-c/GRPCClient/ProtoMethod.m rename to src/objective-c/ProtoRPC/ProtoMethod.m index 4c78b4f61c7..245bf6180b0 100644 --- a/src/objective-c/GRPCClient/ProtoMethod.m +++ b/src/objective-c/ProtoRPC/ProtoMethod.m @@ -35,13 +35,21 @@ @implementation ProtoMethod - (instancetype)initWithPackage:(NSString *)package - interface:(NSString *)interface + service:(NSString *)service method:(NSString *)method { if ((self = [super init])) { _package = [package copy]; - _service = [interface copy]; + _service = [service copy]; _method = [method copy]; } return self; } + +- (NSString *)HTTP2Path { + if (_package) { + return [NSString stringWithFormat:@"/%@.%@/%@", _package, _service, _method]; + } else { + return [NSString stringWithFormat:@"/%@/%@", _service, _method]; + } +} @end diff --git a/src/objective-c/ProtoRPC/ProtoRPC.h b/src/objective-c/ProtoRPC/ProtoRPC.h index c5ab345c9cf..fcc0a507feb 100644 --- a/src/objective-c/ProtoRPC/ProtoRPC.h +++ b/src/objective-c/ProtoRPC/ProtoRPC.h @@ -34,6 +34,8 @@ #import #import +#import "ProtoMethod.h" + @interface ProtoRPC : GRPCCall - (instancetype)initWithHost:(NSString *)host diff --git a/src/objective-c/ProtoRPC/ProtoRPC.m b/src/objective-c/ProtoRPC/ProtoRPC.m index ec38077b714..3a99f1d5ded 100644 --- a/src/objective-c/ProtoRPC/ProtoRPC.m +++ b/src/objective-c/ProtoRPC/ProtoRPC.m @@ -42,15 +42,16 @@ id _responseWriteable; } +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wobjc-designated-initializers" - (instancetype)initWithHost:(NSString *)host - method:(ProtoMethod *)method + path:(NSString *)path requestsWriter:(id)requestsWriter { - return [self initWithHost:host - method:method - requestsWriter:requestsWriter - responseClass:nil - responsesWriteable:nil]; + [NSException raise:NSInvalidArgumentException + format:@"Please use ProtoRPC's designated initializer instead."]; + return nil; } +#pragma clang diagnostic pop // Designated initializer - (instancetype)initWithHost:(NSString *)host @@ -70,7 +71,7 @@ // sending GPBMessages. return [proto data]; }]; - if ((self = [super initWithHost:host method:method requestsWriter:bytesWriter])) { + if ((self = [super initWithHost:host path:method.HTTP2Path requestsWriter:bytesWriter])) { // A writeable that parses the proto messages received. _responseWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { [responsesWriteable writeValue:[responseClass parseFromData:value error:NULL]]; diff --git a/src/objective-c/ProtoRPC/ProtoService.m b/src/objective-c/ProtoRPC/ProtoService.m index e8939389332..d7c5b6a8501 100644 --- a/src/objective-c/ProtoRPC/ProtoService.m +++ b/src/objective-c/ProtoRPC/ProtoService.m @@ -33,10 +33,10 @@ #import "ProtoService.h" -#import #import #import +#import "ProtoMethod.h" #import "ProtoRPC.h" @implementation ProtoService { @@ -70,8 +70,8 @@ responseClass:(Class)responseClass responsesWriteable:(id)responsesWriteable { ProtoMethod *methodName = [[ProtoMethod alloc] initWithPackage:_packageName - interface:_serviceName - method:method]; + service:_serviceName + method:method]; return [[ProtoRPC alloc] initWithHost:_host method:methodName requestsWriter:requestsWriter diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index e3b37b050e4..20a61b7b1cf 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -35,7 +35,7 @@ #import #import -#import +#import #import #import #import @@ -59,21 +59,21 @@ static ProtoMethod *kUnaryCallMethod; - (void)setUp { // This method isn't implemented by the remote server. kInexistentMethod = [[ProtoMethod alloc] initWithPackage:kPackage - interface:kService - method:@"Inexistent"]; + service:kService + method:@"Inexistent"]; kEmptyCallMethod = [[ProtoMethod alloc] initWithPackage:kPackage - interface:kService - method:@"EmptyCall"]; + service:kService + method:@"EmptyCall"]; kUnaryCallMethod = [[ProtoMethod alloc] initWithPackage:kPackage - interface:kService - method:@"UnaryCall"]; + service:kService + method:@"UnaryCall"]; } - (void)testConnectionToRemoteServer { __weak XCTestExpectation *expectation = [self expectationWithDescription:@"Server reachable."]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - method:kInexistentMethod + path:kInexistentMethod.HTTP2Path requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { @@ -95,7 +95,7 @@ static ProtoMethod *kUnaryCallMethod; __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - method:kEmptyCallMethod + path:kEmptyCallMethod.HTTP2Path requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { @@ -123,7 +123,7 @@ static ProtoMethod *kUnaryCallMethod; id requestsWriter = [GRXWriter writerWithValue:[request data]]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - method:kUnaryCallMethod + path:kUnaryCallMethod.HTTP2Path requestsWriter:requestsWriter]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { @@ -153,7 +153,7 @@ static ProtoMethod *kUnaryCallMethod; id requestsWriter = [GRXWriter writerWithValue:[request data]]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - method:kUnaryCallMethod + path:kUnaryCallMethod.HTTP2Path requestsWriter:requestsWriter]; call.requestMetadata[@"Authorization"] = @"Bearer bogusToken"; diff --git a/src/objective-c/tests/LocalClearTextTests.m b/src/objective-c/tests/LocalClearTextTests.m index ace100847fe..d9ad3455aff 100644 --- a/src/objective-c/tests/LocalClearTextTests.m +++ b/src/objective-c/tests/LocalClearTextTests.m @@ -35,7 +35,7 @@ #import #import -#import +#import #import #import #import @@ -88,13 +88,13 @@ static NSString * const kService = @"RouteGuide"; __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; ProtoMethod *method = [[ProtoMethod alloc] initWithPackage:kPackage - interface:kService - method:@"RecordRoute"]; + service:kService + method:@"RecordRoute"]; id requestsWriter = [GRXWriter emptyWriter]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kRouteGuideHost - method:method + path:method.HTTP2Path requestsWriter:requestsWriter]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { @@ -116,8 +116,8 @@ static NSString * const kService = @"RouteGuide"; __weak XCTestExpectation *completion = [self expectationWithDescription:@"RPC completed."]; ProtoMethod *method = [[ProtoMethod alloc] initWithPackage:kPackage - interface:kService - method:@"GetFeature"]; + service:kService + method:@"GetFeature"]; RGDPoint *point = [RGDPoint message]; point.latitude = 28E7; @@ -125,7 +125,7 @@ static NSString * const kService = @"RouteGuide"; id requestsWriter = [GRXWriter writerWithValue:[point data]]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kRouteGuideHost - method:method + path:method.HTTP2Path requestsWriter:requestsWriter]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { From 5260f53cdad8202da3d70e42fa1886e17e8de66b Mon Sep 17 00:00:00 2001 From: Jorge Canizales Date: Sat, 4 Jul 2015 14:47:41 -0700 Subject: [PATCH 54/63] Rename HTTP2Path -> HTTPPath --- src/objective-c/ProtoRPC/ProtoMethod.h | 2 +- src/objective-c/ProtoRPC/ProtoMethod.m | 2 +- src/objective-c/ProtoRPC/ProtoRPC.m | 2 +- src/objective-c/tests/GRPCClientTests.m | 8 ++++---- src/objective-c/tests/LocalClearTextTests.m | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/objective-c/ProtoRPC/ProtoMethod.h b/src/objective-c/ProtoRPC/ProtoMethod.h index 9ecb55a48fc..8f554a04832 100644 --- a/src/objective-c/ProtoRPC/ProtoMethod.h +++ b/src/objective-c/ProtoRPC/ProtoMethod.h @@ -40,7 +40,7 @@ @property(nonatomic, readonly) NSString *service; @property(nonatomic, readonly) NSString *method; -@property(nonatomic, readonly) NSString *HTTP2Path; +@property(nonatomic, readonly) NSString *HTTPPath; - (instancetype)initWithPackage:(NSString *)package service:(NSString *)service diff --git a/src/objective-c/ProtoRPC/ProtoMethod.m b/src/objective-c/ProtoRPC/ProtoMethod.m index 245bf6180b0..1113b4fbaa3 100644 --- a/src/objective-c/ProtoRPC/ProtoMethod.m +++ b/src/objective-c/ProtoRPC/ProtoMethod.m @@ -45,7 +45,7 @@ return self; } -- (NSString *)HTTP2Path { +- (NSString *)HTTPPath { if (_package) { return [NSString stringWithFormat:@"/%@.%@/%@", _package, _service, _method]; } else { diff --git a/src/objective-c/ProtoRPC/ProtoRPC.m b/src/objective-c/ProtoRPC/ProtoRPC.m index 3a99f1d5ded..fe3ccf05413 100644 --- a/src/objective-c/ProtoRPC/ProtoRPC.m +++ b/src/objective-c/ProtoRPC/ProtoRPC.m @@ -71,7 +71,7 @@ // sending GPBMessages. return [proto data]; }]; - if ((self = [super initWithHost:host path:method.HTTP2Path requestsWriter:bytesWriter])) { + if ((self = [super initWithHost:host path:method.HTTPPath requestsWriter:bytesWriter])) { // A writeable that parses the proto messages received. _responseWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { [responsesWriteable writeValue:[responseClass parseFromData:value error:NULL]]; diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 20a61b7b1cf..f9c2d5d8d6e 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -73,7 +73,7 @@ static ProtoMethod *kUnaryCallMethod; __weak XCTestExpectation *expectation = [self expectationWithDescription:@"Server reachable."]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - path:kInexistentMethod.HTTP2Path + path:kInexistentMethod.HTTPPath requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { @@ -95,7 +95,7 @@ static ProtoMethod *kUnaryCallMethod; __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - path:kEmptyCallMethod.HTTP2Path + path:kEmptyCallMethod.HTTPPath requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { @@ -123,7 +123,7 @@ static ProtoMethod *kUnaryCallMethod; id requestsWriter = [GRXWriter writerWithValue:[request data]]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - path:kUnaryCallMethod.HTTP2Path + path:kUnaryCallMethod.HTTPPath requestsWriter:requestsWriter]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { @@ -153,7 +153,7 @@ static ProtoMethod *kUnaryCallMethod; id requestsWriter = [GRXWriter writerWithValue:[request data]]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kHostAddress - path:kUnaryCallMethod.HTTP2Path + path:kUnaryCallMethod.HTTPPath requestsWriter:requestsWriter]; call.requestMetadata[@"Authorization"] = @"Bearer bogusToken"; diff --git a/src/objective-c/tests/LocalClearTextTests.m b/src/objective-c/tests/LocalClearTextTests.m index d9ad3455aff..10c9f13ea3b 100644 --- a/src/objective-c/tests/LocalClearTextTests.m +++ b/src/objective-c/tests/LocalClearTextTests.m @@ -94,7 +94,7 @@ static NSString * const kService = @"RouteGuide"; id requestsWriter = [GRXWriter emptyWriter]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kRouteGuideHost - path:method.HTTP2Path + path:method.HTTPPath requestsWriter:requestsWriter]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { @@ -125,7 +125,7 @@ static NSString * const kService = @"RouteGuide"; id requestsWriter = [GRXWriter writerWithValue:[point data]]; GRPCCall *call = [[GRPCCall alloc] initWithHost:kRouteGuideHost - path:method.HTTP2Path + path:method.HTTPPath requestsWriter:requestsWriter]; id responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { From a82fef14b3caa94c133e9c49504243e28ea2cadd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Jul 2015 08:02:41 -0700 Subject: [PATCH 55/63] Fix bad assert, memory leak --- src/core/client_config/lb_policies/pick_first.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/core/client_config/lb_policies/pick_first.c b/src/core/client_config/lb_policies/pick_first.c index 3d57e3136a4..73da624aff7 100644 --- a/src/core/client_config/lb_policies/pick_first.c +++ b/src/core/client_config/lb_policies/pick_first.c @@ -155,8 +155,6 @@ loop: switch (p->checking_connectivity) { case GRPC_CHANNEL_READY: p->selected = p->subchannels[p->checking_subchannel]; - GPR_ASSERT(grpc_subchannel_check_connectivity(p->selected) == - GRPC_CHANNEL_READY); while ((pp = p->pending_picks)) { p->pending_picks = pp->next; *pp->target = p->selected; @@ -185,6 +183,7 @@ loop: GPR_SWAP(grpc_subchannel *, p->subchannels[p->checking_subchannel], p->subchannels[p->num_subchannels - 1]); p->num_subchannels--; + GRPC_SUBCHANNEL_UNREF(p->subchannels[p->num_subchannels], "pick_first"); if (p->num_subchannels == 0) { while ((pp = p->pending_picks)) { p->pending_picks = pp->next; @@ -197,7 +196,6 @@ loop: p->checking_subchannel %= p->num_subchannels; p->checking_connectivity = grpc_subchannel_check_connectivity( p->subchannels[p->checking_subchannel]); - GRPC_SUBCHANNEL_UNREF(p->subchannels[p->num_subchannels], "pick_first"); add_interested_parties_locked(p); goto loop; } From 825b21cd1f999fa55e6234c2d0d8a6375f252c42 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Jul 2015 08:35:02 -0700 Subject: [PATCH 56/63] Fix segfault The server transport started reading fractionally too early, leading to the accept callback not being ready in some cases. --- src/core/security/server_secure_chttp2.c | 5 +++-- src/core/surface/channel_create.c | 3 ++- src/core/surface/secure_channel_create.c | 6 +++--- src/core/surface/server_chttp2.c | 3 ++- src/core/transport/chttp2_transport.c | 20 ++++++++++++-------- src/core/transport/chttp2_transport.h | 7 +++++-- 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/core/security/server_secure_chttp2.c b/src/core/security/server_secure_chttp2.c index 6a99324da6b..8a7ada07af9 100644 --- a/src/core/security/server_secure_chttp2.c +++ b/src/core/security/server_secure_chttp2.c @@ -99,9 +99,10 @@ static void on_secure_transport_setup_done(void *statep, if (!state->is_shutdown) { mdctx = grpc_mdctx_create(); transport = grpc_create_chttp2_transport( - grpc_server_get_channel_args(state->server), secure_endpoint, NULL, 0, - mdctx, 0); + grpc_server_get_channel_args(state->server), secure_endpoint, mdctx, + 0); setup_transport(state, transport, mdctx); + grpc_chttp2_transport_start_reading(transport, NULL, 0); } else { /* We need to consume this here, because the server may already have gone * away. */ diff --git a/src/core/surface/channel_create.c b/src/core/surface/channel_create.c index 09b4fb782b9..e205f0a9f80 100644 --- a/src/core/surface/channel_create.c +++ b/src/core/surface/channel_create.c @@ -72,7 +72,8 @@ static void connected(void *arg, grpc_endpoint *tcp) { grpc_iomgr_closure *notify; if (tcp != NULL) { c->result->transport = grpc_create_chttp2_transport( - c->args.channel_args, tcp, NULL, 0, c->args.metadata_context, 1); + c->args.channel_args, tcp, c->args.metadata_context, 1); + grpc_chttp2_transport_start_reading(c->result->transport, NULL, 0); GPR_ASSERT(c->result->transport); c->result->filters = gpr_malloc(sizeof(grpc_channel_filter *)); c->result->filters[0] = &grpc_http_client_filter; diff --git a/src/core/surface/secure_channel_create.c b/src/core/surface/secure_channel_create.c index 76fc862621f..34ee3f8400f 100644 --- a/src/core/surface/secure_channel_create.c +++ b/src/core/surface/secure_channel_create.c @@ -82,9 +82,9 @@ static void on_secure_transport_setup_done(void *arg, gpr_log(GPR_ERROR, "Secure transport setup failed with error %d.", status); memset(c->result, 0, sizeof(*c->result)); } else { - c->result->transport = - grpc_create_chttp2_transport(c->args.channel_args, secure_endpoint, - NULL, 0, c->args.metadata_context, 1); + c->result->transport = grpc_create_chttp2_transport( + c->args.channel_args, secure_endpoint, c->args.metadata_context, 1); + grpc_chttp2_transport_start_reading(c->result->transport, NULL, 0); c->result->filters = gpr_malloc(sizeof(grpc_channel_filter *) * 2); c->result->filters[0] = &grpc_client_auth_filter; c->result->filters[1] = &grpc_http_client_filter; diff --git a/src/core/surface/server_chttp2.c b/src/core/surface/server_chttp2.c index 9c02c3ef297..78c53466b38 100644 --- a/src/core/surface/server_chttp2.c +++ b/src/core/surface/server_chttp2.c @@ -61,8 +61,9 @@ static void new_transport(void *server, grpc_endpoint *tcp) { */ grpc_mdctx *mdctx = grpc_mdctx_create(); grpc_transport *transport = grpc_create_chttp2_transport( - grpc_server_get_channel_args(server), tcp, NULL, 0, mdctx, 0); + grpc_server_get_channel_args(server), tcp, mdctx, 0); setup_transport(server, transport, mdctx); + grpc_chttp2_transport_start_reading(transport, NULL, 0); } /* Server callback: start listening on our ports */ diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 0aa28da8f79..054690ac959 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -201,8 +201,8 @@ static void ref_transport(grpc_chttp2_transport *t) { gpr_ref(&t->refs); } static void init_transport(grpc_chttp2_transport *t, const grpc_channel_args *channel_args, - grpc_endpoint *ep, gpr_slice *slices, size_t nslices, - grpc_mdctx *mdctx, int is_client) { + grpc_endpoint *ep, grpc_mdctx *mdctx, + int is_client) { size_t i; int j; @@ -311,9 +311,6 @@ static void init_transport(grpc_chttp2_transport *t, } } } - - REF_TRANSPORT(t, "recv_data"); /* matches unref inside recv_data */ - recv_data(t, slices, nslices, GRPC_ENDPOINT_CB_OK); } static void destroy_transport(grpc_transport *gt) { @@ -1043,9 +1040,16 @@ static const grpc_transport_vtable vtable = { perform_transport_op, destroy_stream, destroy_transport}; grpc_transport *grpc_create_chttp2_transport( - const grpc_channel_args *channel_args, grpc_endpoint *ep, gpr_slice *slices, - size_t nslices, grpc_mdctx *mdctx, int is_client) { + const grpc_channel_args *channel_args, grpc_endpoint *ep, grpc_mdctx *mdctx, + int is_client) { grpc_chttp2_transport *t = gpr_malloc(sizeof(grpc_chttp2_transport)); - init_transport(t, channel_args, ep, slices, nslices, mdctx, is_client); + init_transport(t, channel_args, ep, mdctx, is_client); return &t->base; } + +void grpc_chttp2_transport_start_reading(grpc_transport *transport, + gpr_slice *slices, size_t nslices) { + grpc_chttp2_transport *t = (grpc_chttp2_transport *)transport; + REF_TRANSPORT(t, "recv_data"); /* matches unref inside recv_data */ + recv_data(t, slices, nslices, GRPC_ENDPOINT_CB_OK); +} diff --git a/src/core/transport/chttp2_transport.h b/src/core/transport/chttp2_transport.h index 1747792b95e..fa0d6e4151e 100644 --- a/src/core/transport/chttp2_transport.h +++ b/src/core/transport/chttp2_transport.h @@ -41,7 +41,10 @@ extern int grpc_http_trace; extern int grpc_flowctl_trace; grpc_transport *grpc_create_chttp2_transport( - const grpc_channel_args *channel_args, grpc_endpoint *ep, gpr_slice *slices, - size_t nslices, grpc_mdctx *metadata_context, int is_client); + const grpc_channel_args *channel_args, grpc_endpoint *ep, + grpc_mdctx *metadata_context, int is_client); + +void grpc_chttp2_transport_start_reading(grpc_transport *transport, + gpr_slice *slices, size_t nslices); #endif /* GRPC_INTERNAL_CORE_TRANSPORT_CHTTP2_TRANSPORT_H */ From 3f2db5a4a19427d1c33dec5a1c917ab44b1d2d8b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Jul 2015 10:22:47 -0700 Subject: [PATCH 57/63] Fix potential double delete --- src/core/transport/chttp2/incoming_metadata.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/transport/chttp2/incoming_metadata.c b/src/core/transport/chttp2/incoming_metadata.c index a4b7174329b..68e0912b9ca 100644 --- a/src/core/transport/chttp2/incoming_metadata.c +++ b/src/core/transport/chttp2/incoming_metadata.c @@ -124,6 +124,7 @@ void grpc_incoming_metadata_buffer_move_to_referencing_sopb( sopb->ops[i].data.metadata.list.tail = (void *)(delta + (gpr_intptr)sopb->ops[i].data.metadata.list.tail); } + src->count = 0; } void grpc_chttp2_incoming_metadata_buffer_postprocess_sopb_and_begin_live_op( From f81ac3aced7fecde79458d9ec1b08414ddcdba06 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Jul 2015 09:50:55 -0700 Subject: [PATCH 58/63] Make log lines line up and be prettier --- src/core/support/log_linux.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/core/support/log_linux.c b/src/core/support/log_linux.c index 48349d2c831..7937466b79a 100644 --- a/src/core/support/log_linux.c +++ b/src/core/support/log_linux.c @@ -43,7 +43,9 @@ #ifdef GPR_LINUX +#include #include +#include #include #include #include @@ -71,6 +73,7 @@ void gpr_log(const char *file, int line, gpr_log_severity severity, void gpr_default_log(gpr_log_func_args *args) { char *final_slash; + char *prefix; const char *display_file; char time_buffer[64]; gpr_timespec now = gpr_now(); @@ -89,10 +92,12 @@ void gpr_default_log(gpr_log_func_args *args) { strcpy(time_buffer, "error:strftime"); } - fprintf(stderr, "%s%s.%09d %7ld %s:%d] %s\n", + gpr_asprintf(&prefix, "%s%s.%09d %7tu %s:%d]", gpr_log_severity_string(args->severity), time_buffer, - (int)(now.tv_nsec), gettid(), display_file, args->line, - args->message); + (int)(now.tv_nsec), gettid(), display_file, args->line); + + fprintf(stderr, "%-60s %s\n", prefix, args->message); + gpr_free(prefix); } #endif From 043a0a83c52710ff3b23fe1a0425871456bb1ef2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Jul 2015 11:36:20 -0700 Subject: [PATCH 59/63] Add a short description of available resolution schemes --- src/core/client_config/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/core/client_config/README.md b/src/core/client_config/README.md index 7cb19cd130e..d7aed272233 100644 --- a/src/core/client_config/README.md +++ b/src/core/client_config/README.md @@ -42,3 +42,19 @@ Their behavior is specified by a set of grpc channel filters defined at their construction. To customize this behavior, resolvers build grpc_subchannel_factory objects, which use the decorator pattern to customize construction arguments for concrete grpc_subchannel instances. + + +Naming for GRPC +=============== + +Names in GRPC are represented by a URI. + +The following schemes are currently supported: + +dns:///host:port - dns schemes are currently supported so long as authority is + empty (authority based dns resolution is expected in a future + release) + +unix:path - the unix scheme is used to create and connect to unix domain + sockets - the authority must be empty, and the path represents + the absolute or relative path to the desired socket From 1dd7026cd28f5a64ea8fca766feb16a9cdfcded5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Jul 2015 11:45:30 -0700 Subject: [PATCH 60/63] Fixup some tests --- test/core/bad_client/bad_client.c | 3 ++- test/core/end2end/fixtures/chttp2_socket_pair.c | 8 ++++---- .../fixtures/chttp2_socket_pair_one_byte_at_a_time.c | 8 ++++---- .../end2end/fixtures/chttp2_socket_pair_with_grpc_trace.c | 8 ++++---- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c index 9b6a246075b..8ce666dcde5 100644 --- a/test/core/bad_client/bad_client.c +++ b/test/core/bad_client/bad_client.c @@ -108,8 +108,9 @@ void grpc_run_bad_client_test(grpc_bad_client_server_side_validator validator, a.validator = validator; grpc_server_register_completion_queue(a.server, a.cq); grpc_server_start(a.server); - transport = grpc_create_chttp2_transport(NULL, sfd.server, NULL, 0, mdctx, 0); + transport = grpc_create_chttp2_transport(NULL, sfd.server, mdctx, 0); server_setup_transport(&a, transport, mdctx); + grpc_chttp2_transport_start_reading(transport, NULL, 0); /* Bind everything into the same pollset */ grpc_endpoint_add_to_pollset(sfd.client, grpc_cq_pollset(a.cq)); diff --git a/test/core/end2end/fixtures/chttp2_socket_pair.c b/test/core/end2end/fixtures/chttp2_socket_pair.c index f42b9831c8d..be523608d0a 100644 --- a/test/core/end2end/fixtures/chttp2_socket_pair.c +++ b/test/core/end2end/fixtures/chttp2_socket_pair.c @@ -108,10 +108,10 @@ static void chttp2_init_client_socketpair(grpc_end2end_test_fixture *f, sp_client_setup cs; cs.client_args = client_args; cs.f = f; - transport = - grpc_create_chttp2_transport(client_args, sfd->client, NULL, 0, mdctx, 1); + transport = grpc_create_chttp2_transport(client_args, sfd->client, mdctx, 1); client_setup_transport(&cs, transport, mdctx); GPR_ASSERT(f->client); + grpc_chttp2_transport_start_reading(transport, NULL, 0); } static void chttp2_init_server_socketpair(grpc_end2end_test_fixture *f, @@ -123,9 +123,9 @@ static void chttp2_init_server_socketpair(grpc_end2end_test_fixture *f, f->server = grpc_server_create_from_filters(NULL, 0, server_args); grpc_server_register_completion_queue(f->server, f->cq); grpc_server_start(f->server); - transport = - grpc_create_chttp2_transport(server_args, sfd->server, NULL, 0, mdctx, 0); + transport = grpc_create_chttp2_transport(server_args, sfd->server, mdctx, 0); server_setup_transport(f, transport, mdctx); + grpc_chttp2_transport_start_reading(transport, NULL, 0); } static void chttp2_tear_down_socketpair(grpc_end2end_test_fixture *f) { diff --git a/test/core/end2end/fixtures/chttp2_socket_pair_one_byte_at_a_time.c b/test/core/end2end/fixtures/chttp2_socket_pair_one_byte_at_a_time.c index be520380a70..f875ca54a50 100644 --- a/test/core/end2end/fixtures/chttp2_socket_pair_one_byte_at_a_time.c +++ b/test/core/end2end/fixtures/chttp2_socket_pair_one_byte_at_a_time.c @@ -108,10 +108,10 @@ static void chttp2_init_client_socketpair(grpc_end2end_test_fixture *f, sp_client_setup cs; cs.client_args = client_args; cs.f = f; - transport = - grpc_create_chttp2_transport(client_args, sfd->client, NULL, 0, mdctx, 1); + transport = grpc_create_chttp2_transport(client_args, sfd->client, mdctx, 1); client_setup_transport(&cs, transport, mdctx); GPR_ASSERT(f->client); + grpc_chttp2_transport_start_reading(transport, NULL, 0); } static void chttp2_init_server_socketpair(grpc_end2end_test_fixture *f, @@ -123,9 +123,9 @@ static void chttp2_init_server_socketpair(grpc_end2end_test_fixture *f, f->server = grpc_server_create_from_filters(NULL, 0, server_args); grpc_server_register_completion_queue(f->server, f->cq); grpc_server_start(f->server); - transport = - grpc_create_chttp2_transport(server_args, sfd->server, NULL, 0, mdctx, 0); + transport = grpc_create_chttp2_transport(server_args, sfd->server, mdctx, 0); server_setup_transport(f, transport, mdctx); + grpc_chttp2_transport_start_reading(transport, NULL, 0); } static void chttp2_tear_down_socketpair(grpc_end2end_test_fixture *f) { diff --git a/test/core/end2end/fixtures/chttp2_socket_pair_with_grpc_trace.c b/test/core/end2end/fixtures/chttp2_socket_pair_with_grpc_trace.c index 037281c5ade..52c0e2ca8b2 100644 --- a/test/core/end2end/fixtures/chttp2_socket_pair_with_grpc_trace.c +++ b/test/core/end2end/fixtures/chttp2_socket_pair_with_grpc_trace.c @@ -109,10 +109,10 @@ static void chttp2_init_client_socketpair(grpc_end2end_test_fixture *f, sp_client_setup cs; cs.client_args = client_args; cs.f = f; - transport = - grpc_create_chttp2_transport(client_args, sfd->client, NULL, 0, mdctx, 1); + transport = grpc_create_chttp2_transport(client_args, sfd->client, mdctx, 1); client_setup_transport(&cs, transport, mdctx); GPR_ASSERT(f->client); + grpc_chttp2_transport_start_reading(transport, NULL, 0); } static void chttp2_init_server_socketpair(grpc_end2end_test_fixture *f, @@ -124,9 +124,9 @@ static void chttp2_init_server_socketpair(grpc_end2end_test_fixture *f, f->server = grpc_server_create_from_filters(NULL, 0, server_args); grpc_server_register_completion_queue(f->server, f->cq); grpc_server_start(f->server); - transport = - grpc_create_chttp2_transport(server_args, sfd->server, NULL, 0, mdctx, 0); + transport = grpc_create_chttp2_transport(server_args, sfd->server, mdctx, 0); server_setup_transport(f, transport, mdctx); + grpc_chttp2_transport_start_reading(transport, NULL, 0); } static void chttp2_tear_down_socketpair(grpc_end2end_test_fixture *f) { From 43f5ac6a28f02e9d154b36668717891252e87d64 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Jul 2015 14:09:23 -0700 Subject: [PATCH 61/63] Guard against double pollset shutdown completion --- src/core/iomgr/pollset_posix.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 15ed8e75e6c..12496440ded 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -249,7 +249,8 @@ static void basic_do_promote(void *args, int success) { pollset->in_flight_cbs--; if (pollset->shutting_down) { /* We don't care about this pollset anymore. */ - if (pollset->in_flight_cbs == 0 && pollset->counter == 0) { + if (pollset->in_flight_cbs == 0 && pollset->counter == 0 && !pollset->called_shutdown) { + pollset->called_shutdown = 1; do_shutdown_cb = 1; } } else if (grpc_fd_is_orphaned(fd)) { From bea1e621a120e1e9c1393479599cc9c4c710d648 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Jul 2015 09:53:40 -0700 Subject: [PATCH 62/63] Fix Windows Crash --- src/core/transport/chttp2_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 054690ac959..3483512ab88 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -684,7 +684,7 @@ static void perform_transport_op(grpc_transport *gt, grpc_transport_op *op) { grpc_chttp2_goaway_append( t->global.last_incoming_stream_id, grpc_chttp2_grpc_status_to_http2_error(op->goaway_status), - *op->goaway_message, &t->global.qbuf); + gpr_slice_ref(*op->goaway_message), &t->global.qbuf); } if (op->set_accept_stream != NULL) { From 298b0b93ff764b4c94a165ba0bf6e70f2beb4d6d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 7 Jul 2015 12:44:22 -0700 Subject: [PATCH 63/63] Avoid finishing writes until data actually sent --- src/core/transport/chttp2/writing.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c index fdcc3000991..a78654334e1 100644 --- a/src/core/transport/chttp2/writing.c +++ b/src/core/transport/chttp2/writing.c @@ -97,12 +97,8 @@ int grpc_chttp2_unlocking_check_writes( grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing); } - /* we should either exhaust window or have no ops left, but not both */ - if (stream_global->outgoing_sopb->nops == 0) { - stream_global->outgoing_sopb = NULL; - grpc_chttp2_schedule_closure(transport_global, - stream_global->send_done_closure, 1); - } else if (stream_global->outgoing_window > 0) { + if (stream_global->outgoing_window > 0 && + stream_global->outgoing_sopb->nops != 0) { grpc_chttp2_list_add_writable_stream(transport_global, stream_global); } } @@ -201,6 +197,11 @@ void grpc_chttp2_cleanup_writing( while (grpc_chttp2_list_pop_written_stream( transport_global, transport_writing, &stream_global, &stream_writing)) { + if (stream_global->outgoing_sopb->nops == 0) { + stream_global->outgoing_sopb = NULL; + grpc_chttp2_schedule_closure(transport_global, + stream_global->send_done_closure, 1); + } if (stream_writing->send_closed != GRPC_DONT_SEND_CLOSED) { stream_global->write_state = GRPC_WRITE_STATE_SENT_CLOSE; if (!transport_global->is_client) {