diff --git a/Makefile b/Makefile index 2de8d9a5756..0f3c22a3a84 100644 --- a/Makefile +++ b/Makefile @@ -1017,10 +1017,12 @@ hybrid_end2end_test: $(BINDIR)/$(CONFIG)/hybrid_end2end_test interop_client: $(BINDIR)/$(CONFIG)/interop_client interop_server: $(BINDIR)/$(CONFIG)/interop_server interop_test: $(BINDIR)/$(CONFIG)/interop_test +json_run_localhost: $(BINDIR)/$(CONFIG)/json_run_localhost metrics_client: $(BINDIR)/$(CONFIG)/metrics_client mock_test: $(BINDIR)/$(CONFIG)/mock_test qps_driver: $(BINDIR)/$(CONFIG)/qps_driver qps_interarrival_test: $(BINDIR)/$(CONFIG)/qps_interarrival_test +qps_json_driver: $(BINDIR)/$(CONFIG)/qps_json_driver qps_openloop_test: $(BINDIR)/$(CONFIG)/qps_openloop_test qps_test: $(BINDIR)/$(CONFIG)/qps_test qps_worker: $(BINDIR)/$(CONFIG)/qps_worker @@ -1371,11 +1373,14 @@ buildtests_cxx: buildtests_zookeeper privatelibs_cxx \ $(BINDIR)/$(CONFIG)/interop_client \ $(BINDIR)/$(CONFIG)/interop_server \ $(BINDIR)/$(CONFIG)/interop_test \ + $(BINDIR)/$(CONFIG)/json_run_localhost \ $(BINDIR)/$(CONFIG)/metrics_client \ $(BINDIR)/$(CONFIG)/mock_test \ $(BINDIR)/$(CONFIG)/qps_interarrival_test \ + $(BINDIR)/$(CONFIG)/qps_json_driver \ $(BINDIR)/$(CONFIG)/qps_openloop_test \ $(BINDIR)/$(CONFIG)/qps_test \ + $(BINDIR)/$(CONFIG)/qps_worker \ $(BINDIR)/$(CONFIG)/reconnect_interop_client \ $(BINDIR)/$(CONFIG)/reconnect_interop_server \ $(BINDIR)/$(CONFIG)/secure_auth_context_test \ @@ -1749,7 +1754,7 @@ tools_c: privatelibs_c $(BINDIR)/$(CONFIG)/gen_hpack_tables $(BINDIR)/$(CONFIG)/ tools_cxx: privatelibs_cxx -buildbenchmarks: privatelibs $(BINDIR)/$(CONFIG)/low_level_ping_pong_benchmark $(BINDIR)/$(CONFIG)/qps_driver $(BINDIR)/$(CONFIG)/qps_worker +buildbenchmarks: privatelibs $(BINDIR)/$(CONFIG)/low_level_ping_pong_benchmark $(BINDIR)/$(CONFIG)/qps_driver benchmarks: buildbenchmarks @@ -10719,6 +10724,49 @@ endif endif +JSON_RUN_LOCALHOST_SRC = \ + test/cpp/qps/json_run_localhost.cc \ + +JSON_RUN_LOCALHOST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(JSON_RUN_LOCALHOST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/json_run_localhost: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/json_run_localhost: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/json_run_localhost: $(PROTOBUF_DEP) $(JSON_RUN_LOCALHOST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(JSON_RUN_LOCALHOST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/json_run_localhost + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/qps/json_run_localhost.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_json_run_localhost: $(JSON_RUN_LOCALHOST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(JSON_RUN_LOCALHOST_OBJS:.o=.dep) +endif +endif + + METRICS_CLIENT_SRC = \ $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc \ test/cpp/interop/metrics_client.cc \ @@ -10895,6 +10943,49 @@ endif endif +QPS_JSON_DRIVER_SRC = \ + test/cpp/qps/qps_json_driver.cc \ + +QPS_JSON_DRIVER_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(QPS_JSON_DRIVER_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/qps_json_driver: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/qps_json_driver: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/qps_json_driver: $(PROTOBUF_DEP) $(QPS_JSON_DRIVER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(QPS_JSON_DRIVER_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/qps_json_driver + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_json_driver.o: $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_qps_json_driver: $(QPS_JSON_DRIVER_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(QPS_JSON_DRIVER_OBJS:.o=.dep) +endif +endif + + QPS_OPENLOOP_TEST_SRC = \ test/cpp/qps/qps_openloop_test.cc \ diff --git a/build.yaml b/build.yaml index a3af1eeb0cd..027cd34a7e3 100644 --- a/build.yaml +++ b/build.yaml @@ -2593,6 +2593,20 @@ targets: - mac - linux - posix +- name: json_run_localhost + build: test + run: false + language: c++ + src: + - test/cpp/qps/json_run_localhost.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config - name: metrics_client build: test run: false @@ -2652,6 +2666,21 @@ targets: - mac - linux - posix +- name: qps_json_driver + build: test + run: false + language: c++ + src: + - test/cpp/qps/qps_json_driver.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config - name: qps_openloop_test cpu_cost: 0.5 build: test @@ -2691,7 +2720,8 @@ targets: - linux - posix - name: qps_worker - build: benchmark + build: test + run: false language: c++ headers: - test/cpp/qps/client.h diff --git a/src/proto/grpc/testing/control.proto b/src/proto/grpc/testing/control.proto index 11e752a74b2..458b19c4d6c 100644 --- a/src/proto/grpc/testing/control.proto +++ b/src/proto/grpc/testing/control.proto @@ -169,3 +169,28 @@ message CoreResponse { message Void { } + +// A single performance scenario: input to qps_json_driver +message Scenario { + // Human readable name for this scenario + string name = 1; + // Client configuration + ClientConfig client_config = 2; + // Number of clients to start for the test + int32 num_clients = 3; + // Server configuration + ServerConfig server_config = 4; + // Number of servers to start for the test + int32 num_servers = 5; + // Warmup period, in seconds + int32 warmup_seconds = 6; + // Benchmark time, in seconds + int32 benchmark_seconds = 7; + // Number of workers to spawn locally (usually zero) + int32 spawn_local_worker_count = 8; +} + +// A set of scenarios to be run with qps_json_driver +message Scenarios { + repeated Scenario scenarios = 1; +} diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py new file mode 100755 index 00000000000..9d6bf2ab73c --- /dev/null +++ b/test/cpp/qps/gen_build_yaml.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python2.7 + +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json +import pipes +import shutil +import sys +import os +import yaml + +run_tests_root = os.path.abspath(os.path.join( + os.path.dirname(sys.argv[0]), + '../../../tools/run_tests')) +sys.path.append(run_tests_root) + +import performance.scenario_config as scenario_config + +print yaml.dump({ + 'tests': [ + { + 'name': 'json_run_localhost', + 'shortname': 'json_run_localhost:%s' % js['name'], + 'args': ['--scenario_json', pipes.quote(json.dumps(js))], + 'ci_platforms': ['linux', 'mac', 'posix', 'windows'], + 'platforms': ['linux', 'mac', 'posix', 'windows'], + 'flaky': False, + 'language': 'c++', + 'boringssl': True, + 'defaults': 'boringssl', + 'cpu_cost': 1000.0, + 'exclude_configs': [] + } + for js in scenario_config.CXXLanguage().scenarios() + ] +}) diff --git a/test/cpp/qps/json_run_localhost.cc b/test/cpp/qps/json_run_localhost.cc new file mode 100644 index 00000000000..6545dc2917d --- /dev/null +++ b/test/cpp/qps/json_run_localhost.cc @@ -0,0 +1,86 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include + +#include "src/core/lib/support/env.h" +#include "test/core/util/port.h" +#include "test/cpp/util/subprocess.h" + +using grpc::SubProcess; + +template +std::string as_string(const T& val) { + std::ostringstream out; + out << val; + return out.str(); +} + +int main(int argc, char** argv) { + typedef std::unique_ptr SubProcessPtr; + std::vector jobs; + + std::string my_bin = argv[0]; + std::string bin_dir = my_bin.substr(0, my_bin.rfind('/')); + + std::ostringstream env; + bool first = true; + + for (int i = 0; i < 2; i++) { + auto port = grpc_pick_unused_port_or_die(); + std::vector args = {bin_dir + "/qps_worker", "-driver_port", + as_string(port)}; + jobs.emplace_back(new SubProcess(args)); + if (!first) env << ","; + env << "localhost:" << port; + first = false; + } + + gpr_setenv("QPS_WORKERS", env.str().c_str()); + std::vector args = {bin_dir + "/qps_json_driver"}; + for (int i = 1; i < argc; i++) { + args.push_back(argv[i]); + } + SubProcess(args).Join(); + + for (auto it = jobs.begin(); it != jobs.end(); ++it) { + (*it)->Interrupt(); + } + for (auto it = jobs.begin(); it != jobs.end(); ++it) { + (*it)->Join(); + } +} diff --git a/test/cpp/qps/qps_json_driver.cc b/test/cpp/qps/qps_json_driver.cc new file mode 100644 index 00000000000..8943a43ba87 --- /dev/null +++ b/test/cpp/qps/qps_json_driver.cc @@ -0,0 +1,127 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +#include +#include + +#include +#include + +#include "test/cpp/qps/driver.h" +#include "test/cpp/qps/report.h" +#include "test/cpp/util/benchmark_config.h" + +DEFINE_string(scenarios_file, "", + "JSON file containing an array of Scenario objects"); +DEFINE_string(scenarios_json, "", + "JSON string containing an array of Scenario objects"); + +namespace grpc { +namespace testing { + +static void QpsDriver() { + grpc::string json; + + if (FLAGS_scenarios_file != "") { + if (FLAGS_scenarios_json != "") { + gpr_log(GPR_ERROR, + "Only one of --scenarios_file or --scenarios_json must be set"); + abort(); + } + // Read the json data from disk + FILE *json_file = fopen(FLAGS_scenarios_file.c_str(), "r"); + GPR_ASSERT(json_file != NULL); + fseek(json_file, 0, SEEK_END); + long len = ftell(json_file); + char *data = new char[len]; + fseek(json_file, 0, SEEK_SET); + GPR_ASSERT(len == (long)fread(data, 1, len, json_file)); + fclose(json_file); + json = grpc::string(data, data + len); + delete[] data; + } else if (FLAGS_scenarios_json != "") { + json = FLAGS_scenarios_json.c_str(); + } else { + gpr_log(GPR_ERROR, + "One of --scenarios_file or --scenarios_json must be set"); + abort(); + } + + // Parse into an array of scenarios + Scenarios scenarios; + std::unique_ptr type_resolver( + google::protobuf::util::NewTypeResolverForDescriptorPool( + "type.googleapis.com", + google::protobuf::DescriptorPool::generated_pool())); + grpc::string binary; + auto status = JsonToBinaryString(type_resolver.get(), + "type.googleapis.com/grpc.testing.Scenarios", + json, &binary); + if (!status.ok()) { + grpc::string msg(status.error_message()); + gpr_log(GPR_ERROR, "Failed to convert json to binary: errcode=%d msg=%s", + status.error_code(), msg.c_str()); + gpr_log(GPR_ERROR, "JSON: ", json.c_str()); + abort(); + } + GPR_ASSERT(scenarios.ParseFromString(binary)); + + for (int i = 0; i < scenarios.scenarios_size(); i++) { + const Scenario &scenario = scenarios.scenarios(i); + std::cerr << "RUNNING SCENARIO: " << scenario.name() << "\n"; + const auto result = + RunScenario(scenario.client_config(), scenario.num_clients(), + scenario.server_config(), scenario.num_servers(), + scenario.warmup_seconds(), scenario.benchmark_seconds(), + scenario.spawn_local_worker_count()); + + GetReporter()->ReportQPS(*result); + GetReporter()->ReportQPSPerCore(*result); + GetReporter()->ReportLatency(*result); + GetReporter()->ReportTimes(*result); + } +} + +} // namespace testing +} // namespace grpc + +int main(int argc, char **argv) { + grpc::testing::InitBenchmark(&argc, &argv, true); + + grpc::testing::QpsDriver(); + + return 0; +} diff --git a/test/cpp/util/subprocess.cc b/test/cpp/util/subprocess.cc index c31802b9a3b..00c6244a4f1 100644 --- a/test/cpp/util/subprocess.cc +++ b/test/cpp/util/subprocess.cc @@ -39,15 +39,15 @@ namespace grpc { -static gpr_subprocess *MakeProcess(std::initializer_list args) { - std::vector vargs; +static gpr_subprocess* MakeProcess(const std::vector& args) { + std::vector vargs; for (auto it = args.begin(); it != args.end(); ++it) { vargs.push_back(it->c_str()); } return gpr_subprocess_create(vargs.size(), &vargs[0]); } -SubProcess::SubProcess(std::initializer_list args) +SubProcess::SubProcess(const std::vector& args) : subprocess_(MakeProcess(args)) {} SubProcess::~SubProcess() { gpr_subprocess_destroy(subprocess_); } diff --git a/test/cpp/util/subprocess.h b/test/cpp/util/subprocess.h index 8fafe3d1772..f7b553ae030 100644 --- a/test/cpp/util/subprocess.h +++ b/test/cpp/util/subprocess.h @@ -36,6 +36,7 @@ #include #include +#include struct gpr_subprocess; @@ -43,7 +44,7 @@ namespace grpc { class SubProcess { public: - SubProcess(std::initializer_list args); + SubProcess(const std::vector& args); ~SubProcess(); int Join(); diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index c6317130496..9a1a7a72491 100644 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -34,7 +34,8 @@ gen_build_yaml_dirs=" \ src/zlib \ test/core/bad_client \ test/core/bad_ssl \ - test/core/end2end" + test/core/end2end \ + test/cpp/qps" gen_build_files="" for gen_build_yaml in $gen_build_yaml_dirs do @@ -42,4 +43,3 @@ do $gen_build_yaml/gen_build_yaml.py > $output_file gen_build_files="$gen_build_files $output_file" done - diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh index b33ee3a58c0..2c962cba37d 100755 --- a/tools/run_tests/performance/build_performance.sh +++ b/tools/run_tests/performance/build_performance.sh @@ -41,7 +41,7 @@ CONFIG=${CONFIG:-opt} # TODO(jtattermusch): not embedding OpenSSL breaks the C# build because # grpc_csharp_ext needs OpenSSL embedded and some intermediate files from # this build will be reused. -make CONFIG=${CONFIG} EMBED_OPENSSL=true EMBED_ZLIB=true qps_worker qps_driver -j8 +make CONFIG=${CONFIG} EMBED_OPENSSL=true EMBED_ZLIB=true qps_worker qps_driver qps_json_driver -j8 for language in $@ do @@ -50,4 +50,3 @@ do tools/run_tests/run_tests.py -l $language -c $CONFIG --build_only -j 8 fi done - diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index f95e531fa23..7a82d257e40 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -29,6 +29,37 @@ # performance scenario configuration for various languages +SINGLE_MACHINE_CORES=8 +WARMUP_SECONDS=5 +BENCHMARK_SECONDS=30 + +EMPTY_GENERIC_PAYLOAD = { + 'bytebuf_params': { + 'req_size': 0, + 'resp_size': 0, + } +} +EMPTY_PROTO_PAYLOAD = { + 'simple_params': { + 'req_size': 0, + 'resp_size': 0, + } +} +BIG_GENERIC_PAYLOAD = { + 'bytebuf_params': { + 'req_size': 65536, + 'resp_size': 65536, + } +} + +# deep is the number of RPCs outstanding on a channel in non-ping-pong tests +# (the value used is 1 otherwise) +DEEP=100 + +# wide is the number of client channels in multi-channel tests (1 otherwise) +WIDE=64 + + class CXXLanguage: def __init__(self): @@ -41,38 +72,178 @@ class CXXLanguage: return 0 def scenarios(self): - # TODO(jtattermusch): add more scenarios - return { - # Scenario 1: generic async streaming ping-pong (contentionless latency) - 'cpp_async_generic_streaming_ping_pong': [ - '--rpc_type=STREAMING', - '--client_type=ASYNC_CLIENT', - '--server_type=ASYNC_GENERIC_SERVER', - '--outstanding_rpcs_per_channel=1', - '--client_channels=1', - '--bbuf_req_size=0', - '--bbuf_resp_size=0', - '--async_client_threads=1', - '--async_server_threads=1', - '--secure_test=true', - '--num_servers=1', - '--num_clients=1', - '--server_core_limit=0', - '--client_core_limit=0'], - # Scenario 5: Sync unary ping-pong with protobufs - 'cpp_sync_unary_ping_pong_protobuf': [ - '--rpc_type=UNARY', - '--client_type=SYNC_CLIENT', - '--server_type=SYNC_SERVER', - '--outstanding_rpcs_per_channel=1', - '--client_channels=1', - '--simple_req_size=0', - '--simple_resp_size=0', - '--secure_test=true', - '--num_servers=1', - '--num_clients=1', - '--server_core_limit=0', - '--client_core_limit=0']} + # TODO(ctiller): add 70% load latency test + for secure in [True, False]: + if secure: + secstr = 'secure' + secargs = {'use_test_ca': True, + 'server_host_override': 'foo.test.google.fr'} + else: + secstr = 'insecure' + secargs = None + + yield { + 'name': 'generic_async_streaming_ping_pong_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 1, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'generic_async_streaming_qps_unconstrained_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 0, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': DEEP, + 'client_channels': WIDE, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 1, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'generic_async_streaming_qps_one_server_core_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 0, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': DEEP, + 'client_channels': WIDE, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': 1, + 'async_server_threads': 1, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'protobuf_async_qps_unconstrained_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 0, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': DEEP, + 'client_channels': WIDE, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 1, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'single_channel_throughput_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': BIG_GENERIC_PAYLOAD, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 1, + 'payload_config': BIG_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'protobuf_async_ping_pong_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 1, + 'payload_config': EMPTY_PROTO_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } def __str__(self): return 'c++' @@ -91,23 +262,32 @@ class CSharpLanguage: def scenarios(self): # TODO(jtattermusch): add more scenarios - return { - # Scenario 1: generic async streaming ping-pong (contentionless latency) - 'csharp_async_generic_streaming_ping_pong': [ - '--rpc_type=STREAMING', - '--client_type=ASYNC_CLIENT', - '--server_type=ASYNC_GENERIC_SERVER', - '--outstanding_rpcs_per_channel=1', - '--client_channels=1', - '--bbuf_req_size=0', - '--bbuf_resp_size=0', - '--async_client_threads=1', - '--async_server_threads=1', - '--secure_test=true', - '--num_servers=1', - '--num_clients=1', - '--server_core_limit=0', - '--client_core_limit=0']} + yield { + 'name': 'csharp_async_generic_streaming_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 1, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } def __str__(self): return 'csharp' @@ -127,20 +307,32 @@ class NodeLanguage: def scenarios(self): # TODO(jtattermusch): add more scenarios - return { - 'node_sync_unary_ping_pong_protobuf': [ - '--rpc_type=UNARY', - '--client_type=ASYNC_CLIENT', - '--server_type=ASYNC_SERVER', - '--outstanding_rpcs_per_channel=1', - '--client_channels=1', - '--simple_req_size=0', - '--simple_resp_size=0', - '--secure_test=false', - '--num_servers=1', - '--num_clients=1', - '--server_core_limit=0', - '--client_core_limit=0']} + yield { + 'name': 'node_sync_unary_ping_pong_protobuf', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 1, + 'payload_config': EMPTY_PROTO_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } def __str__(self): return 'node' diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index e1268e2ecbb..51ed35f7608 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -33,8 +33,10 @@ import argparse import itertools import jobset +import json import multiprocessing import os +import pipes import subprocess import sys import tempfile @@ -87,18 +89,34 @@ def create_qpsworker_job(language, shortname=None, return QpsWorkerJob(jobspec, language, host_and_port) -def create_scenario_jobspec(scenario_name, driver_args, workers, remote_host=None): +def create_scenario_jobspec(scenario_json, workers, remote_host=None): """Runs one scenario using QPS driver.""" # setting QPS_WORKERS env variable here makes sure it works with SSH too. - cmd = 'QPS_WORKERS="%s" bins/opt/qps_driver ' % ','.join(workers) - cmd += ' '.join(driver_args) + cmd = 'QPS_WORKERS="%s" bins/opt/qps_json_driver ' % ','.join(workers) + cmd += '--scenarios_json=%s' % pipes.quote(json.dumps({'scenarios': [scenario_json]})) if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && %s"' % (user_at_host, cmd) return jobset.JobSpec( cmdline=[cmd], - shortname='qps_driver.%s' % scenario_name, + shortname='qps_json_driver.%s' % scenario_json['name'], + timeout_seconds=3*60, + shell=True, + verbose_success=True) + + +def create_quit_jobspec(workers, remote_host=None): + """Runs quit using QPS driver.""" + # setting QPS_WORKERS env variable here makes sure it works with SSH too. + cmd = 'QPS_WORKERS="%s" bins/opt/qps_driver --quit' % ','.join(workers) + if remote_host: + user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) + cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && %s"' % (user_at_host, cmd) + + return jobset.JobSpec( + cmdline=[cmd], + shortname='qps_driver.quit', timeout_seconds=3*60, shell=True, verbose_success=True) @@ -207,9 +225,8 @@ def create_scenarios(languages, workers_by_lang, remote_host=None): """Create jobspecs for scenarios to run.""" scenarios = [] for language in languages: - for scenario_name, driver_args in language.scenarios().iteritems(): - scenario = create_scenario_jobspec(scenario_name, - driver_args, + for scenario_json in language.scenarios(): + scenario = create_scenario_jobspec(scenario_json, workers_by_lang[str(language)], remote_host=remote_host) scenarios.append(scenario) @@ -218,10 +235,7 @@ def create_scenarios(languages, workers_by_lang, remote_host=None): all_workers = [worker for workers in workers_by_lang.values() for worker in workers] - scenarios.append(create_scenario_jobspec('quit_workers', - ['--quit=true'], - all_workers, - remote_host=remote_host)) + scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host)) return scenarios diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index a13f2a3a355..dfac94660b6 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -192,7 +192,7 @@ class CLanguage(object): else: cmdline = [binary] + target['args'] out.append(self.config.job_spec(cmdline, [binary], - shortname=' '.join(cmdline), + shortname=target.get('shortname', ' '.join(cmdline)), cpu_cost=target['cpu_cost'], flaky=target.get('flaky', False), environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index ba4db96b566..afe35cac9c7 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -2223,6 +2223,25 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [], + "language": "c++", + "name": "json_run_localhost", + "src": [ + "test/cpp/qps/json_run_localhost.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -2301,6 +2320,26 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util", + "qps" + ], + "headers": [], + "language": "c++", + "name": "qps_json_driver", + "src": [ + "test/cpp/qps/qps_json_driver.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 88aae70aef5..3fff81d9601 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -21510,6 +21510,318 @@ "posix" ] }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:generic_async_streaming_ping_pong_secure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:generic_async_streaming_qps_unconstrained_secure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:generic_async_streaming_qps_one_server_core_secure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"protobuf_async_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:protobuf_async_qps_unconstrained_secure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"single_channel_throughput_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:single_channel_throughput_secure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"protobuf_async_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:protobuf_async_ping_pong_secure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:generic_async_streaming_ping_pong_insecure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:generic_async_streaming_qps_unconstrained_insecure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:generic_async_streaming_qps_one_server_core_insecure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"protobuf_async_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:protobuf_async_qps_unconstrained_insecure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"single_channel_throughput_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:single_channel_throughput_insecure" + }, + { + "args": [ + "--scenario_json", + "'{\"name\": \"protobuf_async_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + ], + "boringssl": true, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1000.0, + "defaults": "boringssl", + "exclude_configs": [], + "flaky": false, + "language": "c++", + "name": "json_run_localhost", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "shortname": "json_run_localhost:protobuf_async_ping_pong_insecure" + }, { "args": [ "test/core/transport/chttp2/hpack_parser_corpus/0141fcddc9807ee093313b2256f1306fbbdc6cda" diff --git a/vsprojects/vcxproj/test/json_run_localhost/json_run_localhost.vcxproj b/vsprojects/vcxproj/test/json_run_localhost/json_run_localhost.vcxproj new file mode 100644 index 00000000000..34507b656ea --- /dev/null +++ b/vsprojects/vcxproj/test/json_run_localhost/json_run_localhost.vcxproj @@ -0,0 +1,210 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {91678827-DAEF-2E2F-9CD1-1F5E5DD54842} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + json_run_localhost + static + Debug + static + Debug + + + json_run_localhost + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {0BE77741-552A-929B-A497-4EF7ECE17A64} + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + {3F7D093D-11F9-C4BC-BEB7-18EB28E3F290} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/json_run_localhost/json_run_localhost.vcxproj.filters b/vsprojects/vcxproj/test/json_run_localhost/json_run_localhost.vcxproj.filters new file mode 100644 index 00000000000..84c5d1e377d --- /dev/null +++ b/vsprojects/vcxproj/test/json_run_localhost/json_run_localhost.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\cpp\qps + + + + + + {6a0e8372-94ed-67cd-edda-56ba97debf76} + + + {2a065f28-e35d-1a75-6e12-fd86e7b99443} + + + {6eec400a-a3a2-2904-8b49-92aeb69c4448} + + + + diff --git a/vsprojects/vcxproj/test/qps_json_driver/qps_json_driver.vcxproj b/vsprojects/vcxproj/test/qps_json_driver/qps_json_driver.vcxproj new file mode 100644 index 00000000000..d1dea3ec4ae --- /dev/null +++ b/vsprojects/vcxproj/test/qps_json_driver/qps_json_driver.vcxproj @@ -0,0 +1,213 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {79B45023-E5EB-4DF4-BBED-525CC0ACD832} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + qps_json_driver + static + Debug + static + Debug + + + qps_json_driver + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {8423B0C5-2428-CA10-82EF-7B5C1F3D8011} + + + {0BE77741-552A-929B-A497-4EF7ECE17A64} + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + {3F7D093D-11F9-C4BC-BEB7-18EB28E3F290} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/qps_json_driver/qps_json_driver.vcxproj.filters b/vsprojects/vcxproj/test/qps_json_driver/qps_json_driver.vcxproj.filters new file mode 100644 index 00000000000..62b9be85cc3 --- /dev/null +++ b/vsprojects/vcxproj/test/qps_json_driver/qps_json_driver.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\cpp\qps + + + + + + {51cad99a-2bf1-0e37-a1be-0c9ad02355d3} + + + {f8692949-7450-0a31-11fc-a17ea63b6357} + + + {73b4c157-7352-ba56-4790-66e27fa90451} + + + + diff --git a/vsprojects/vcxproj/test/qps_worker/qps_worker.vcxproj b/vsprojects/vcxproj/test/qps_worker/qps_worker.vcxproj new file mode 100644 index 00000000000..0ed96fcf8f3 --- /dev/null +++ b/vsprojects/vcxproj/test/qps_worker/qps_worker.vcxproj @@ -0,0 +1,217 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {89ACCD5A-8411-634E-BF28-7115F46A807D} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + qps_worker + static + Debug + static + Debug + + + qps_worker + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + + + + + {8423B0C5-2428-CA10-82EF-7B5C1F3D8011} + + + {0BE77741-552A-929B-A497-4EF7ECE17A64} + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + {3F7D093D-11F9-C4BC-BEB7-18EB28E3F290} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/qps_worker/qps_worker.vcxproj.filters b/vsprojects/vcxproj/test/qps_worker/qps_worker.vcxproj.filters new file mode 100644 index 00000000000..a1f23908271 --- /dev/null +++ b/vsprojects/vcxproj/test/qps_worker/qps_worker.vcxproj.filters @@ -0,0 +1,29 @@ + + + + + test\cpp\qps + + + + + test\cpp\qps + + + test\cpp\qps + + + + + + {bc4b8555-fcc7-4f70-96d9-581f535382e1} + + + {79b85b26-c29a-b8a0-06a4-d4e44f2350aa} + + + {1c2b326f-ab54-97da-d40e-b0e99dd76de6} + + + +