|
|
|
# Copyright 2016 gRPC authors.
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
|
|
|
# performance scenario configuration for various languages
|
|
|
|
|
|
|
|
import math
|
|
|
|
|
|
|
|
WARMUP_SECONDS = 5
|
|
|
|
JAVA_WARMUP_SECONDS = 15 # Java needs more warmup time for JIT to kick in.
|
|
|
|
CXX_WARMUP_SECONDS = 30 # Core needs more warmup time for the thread pool to scale appropriately.
|
|
|
|
BENCHMARK_SECONDS = 30
|
|
|
|
|
|
|
|
SMOKETEST = "smoketest"
|
|
|
|
SCALABLE = "scalable"
|
|
|
|
INPROC = "inproc"
|
|
|
|
SWEEP = "sweep"
|
|
|
|
PSM = "psm"
|
|
|
|
# A small superset of the benchmarks required to produce
|
|
|
|
# https://grafana-dot-grpc-testing.appspot.com/
|
|
|
|
DASHBOARD = "dashboard"
|
|
|
|
DEFAULT_CATEGORIES = (SCALABLE, SMOKETEST)
|
|
|
|
|
|
|
|
SECURE_SECARGS = {
|
|
|
|
"use_test_ca": True,
|
|
|
|
"server_host_override": "foo.test.google.fr",
|
|
|
|
}
|
|
|
|
|
|
|
|
HISTOGRAM_PARAMS = {
|
|
|
|
"resolution": 0.01,
|
|
|
|
"max_possible": 60e9,
|
|
|
|
}
|
|
|
|
|
|
|
|
# target number of RPCs outstanding on across all client channels in
|
|
|
|
# non-ping-pong tests (since we can only specify per-channel numbers, the
|
|
|
|
# actual target will be slightly higher)
|
|
|
|
OUTSTANDING_REQUESTS = {"async": 6400, "async-limited": 800, "sync": 1000}
|
|
|
|
|
|
|
|
# wide is the number of client channels in multi-channel tests (1 otherwise)
|
|
|
|
WIDE = 64
|
|
|
|
|
|
|
|
|
|
|
|
def _get_secargs(is_secure):
|
|
|
|
if is_secure:
|
|
|
|
return SECURE_SECARGS
|
|
|
|
else:
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
def remove_nonproto_fields(scenario):
|
|
|
|
"""Removes special-purpose fields that don't belong in the protobuf.
|
|
|
|
|
|
|
|
This function removes additional information about the scenario that is not
|
|
|
|
included in the ScenarioConfig protobuf message.
|
|
|
|
"""
|
|
|
|
scenario.pop("CATEGORIES", None)
|
|
|
|
scenario.pop("CLIENT_LANGUAGE", None)
|
|
|
|
scenario.pop("SERVER_LANGUAGE", None)
|
|
|
|
scenario.pop("EXCLUDED_POLL_ENGINES", None)
|
|
|
|
return scenario
|
|
|
|
|
|
|
|
|
|
|
|
def geometric_progression(start, stop, step):
|
|
|
|
n = start
|
|
|
|
while n < stop:
|
|
|
|
yield int(round(n))
|
|
|
|
n *= step
|
|
|
|
|
|
|
|
|
|
|
|
def _payload_type(use_generic_payload, req_size, resp_size):
|
|
|
|
r = {}
|
|
|
|
sizes = {
|
|
|
|
"req_size": req_size,
|
|
|
|
"resp_size": resp_size,
|
|
|
|
}
|
|
|
|
if use_generic_payload:
|
|
|
|
r["bytebuf_params"] = sizes
|
|
|
|
else:
|
|
|
|
r["simple_params"] = sizes
|
|
|
|
return r
|
|
|
|
|
|
|
|
|
|
|
|
def _load_params(offered_load):
|
|
|
|
r = {}
|
|
|
|
if offered_load is None:
|
|
|
|
r["closed_loop"] = {}
|
|
|
|
else:
|
|
|
|
load = {}
|
|
|
|
load["offered_load"] = offered_load
|
|
|
|
r["poisson"] = load
|
|
|
|
return r
|
|
|
|
|
|
|
|
|
|
|
|
def _add_channel_arg(config, key, value):
|
|
|
|
if "channel_args" in config:
|
|
|
|
channel_args = config["channel_args"]
|
|
|
|
else:
|
|
|
|
channel_args = []
|
|
|
|
config["channel_args"] = channel_args
|
|
|
|
arg = {"name": key}
|
|
|
|
if isinstance(value, int):
|
|
|
|
arg["int_value"] = value
|
|
|
|
else:
|
|
|
|
arg["str_value"] = value
|
|
|
|
channel_args.append(arg)
|
|
|
|
|
|
|
|
|
|
|
|
def _ping_pong_scenario(
|
|
|
|
name,
|
|
|
|
rpc_type,
|
|
|
|
client_type,
|
|
|
|
server_type,
|
|
|
|
secure=True,
|
|
|
|
use_generic_payload=False,
|
|
|
|
req_size=0,
|
|
|
|
resp_size=0,
|
|
|
|
unconstrained_client=None,
|
|
|
|
client_language=None,
|
|
|
|
server_language=None,
|
|
|
|
async_server_threads=0,
|
|
|
|
client_processes=0,
|
|
|
|
server_processes=0,
|
|
|
|
server_threads_per_cq=0,
|
|
|
|
client_threads_per_cq=0,
|
|
|
|
warmup_seconds=WARMUP_SECONDS,
|
|
|
|
categories=None,
|
|
|
|
channels=None,
|
|
|
|
outstanding=None,
|
|
|
|
num_clients=None,
|
|
|
|
resource_quota_size=None,
|
|
|
|
messages_per_stream=None,
|
|
|
|
excluded_poll_engines=None,
|
|
|
|
minimal_stack=False,
|
|
|
|
offered_load=None,
|
|
|
|
):
|
|
|
|
"""Creates a basic ping pong scenario."""
|
|
|
|
scenario = {
|
|
|
|
"name": name,
|
|
|
|
"num_servers": 1,
|
|
|
|
"num_clients": 1,
|
|
|
|
"client_config": {
|
|
|
|
"client_type": client_type,
|
|
|
|
"security_params": _get_secargs(secure),
|
|
|
|
"outstanding_rpcs_per_channel": 1,
|
|
|
|
"client_channels": 1,
|
|
|
|
"async_client_threads": 1,
|
|
|
|
"client_processes": client_processes,
|
|
|
|
"threads_per_cq": client_threads_per_cq,
|
|
|
|
"rpc_type": rpc_type,
|
|
|
|
"histogram_params": HISTOGRAM_PARAMS,
|
|
|
|
"channel_args": [],
|
|
|
|
},
|
|
|
|
"server_config": {
|
|
|
|
"server_type": server_type,
|
|
|
|
"security_params": _get_secargs(secure),
|
|
|
|
"async_server_threads": async_server_threads,
|
|
|
|
"server_processes": server_processes,
|
|
|
|
"threads_per_cq": server_threads_per_cq,
|
|
|
|
"channel_args": [],
|
|
|
|
},
|
|
|
|
"warmup_seconds": warmup_seconds,
|
|
|
|
"benchmark_seconds": BENCHMARK_SECONDS,
|
|
|
|
"CATEGORIES": list(DEFAULT_CATEGORIES),
|
|
|
|
"EXCLUDED_POLL_ENGINES": [],
|
|
|
|
}
|
|
|
|
if resource_quota_size:
|
|
|
|
scenario["server_config"]["resource_quota_size"] = resource_quota_size
|
|
|
|
if use_generic_payload:
|
|
|
|
if server_type != "ASYNC_GENERIC_SERVER":
|
|
|
|
raise Exception("Use ASYNC_GENERIC_SERVER for generic payload.")
|
|
|
|
scenario["server_config"]["payload_config"] = _payload_type(
|
|
|
|
use_generic_payload, req_size, resp_size
|
|
|
|
)
|
|
|
|
|
|
|
|
scenario["client_config"]["payload_config"] = _payload_type(
|
|
|
|
use_generic_payload, req_size, resp_size
|
|
|
|
)
|
|
|
|
|
|
|
|
# Optimization target of 'throughput' does not work well with epoll1 polling
|
|
|
|
# engine. Use the default value of 'blend'
|
|
|
|
optimization_target = "throughput"
|
|
|
|
|
|
|
|
if unconstrained_client:
|
|
|
|
outstanding_calls = (
|
|
|
|
outstanding
|
|
|
|
if outstanding is not None
|
|
|
|
else OUTSTANDING_REQUESTS[unconstrained_client]
|
|
|
|
)
|
|
|
|
# clamp buffer usage to something reasonable (16 gig for now)
|
|
|
|
MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024
|
|
|
|
if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE:
|
|
|
|
outstanding_calls = max(
|
|
|
|
1, MAX_MEMORY_USE / max(req_size, resp_size)
|
|
|
|
)
|
|
|
|
wide = channels if channels is not None else WIDE
|
|
|
|
deep = int(math.ceil(1.0 * outstanding_calls / wide))
|
|
|
|
|
|
|
|
scenario["num_clients"] = (
|
|
|
|
num_clients if num_clients is not None else 0
|
|
|
|
) # use as many clients as available.
|
|
|
|
scenario["client_config"]["outstanding_rpcs_per_channel"] = deep
|
|
|
|
scenario["client_config"]["client_channels"] = wide
|
|
|
|
scenario["client_config"]["async_client_threads"] = 0
|
|
|
|
if offered_load is not None:
|
|
|
|
optimization_target = "latency"
|
|
|
|
else:
|
|
|
|
scenario["client_config"]["outstanding_rpcs_per_channel"] = 1
|
|
|
|
scenario["client_config"]["client_channels"] = 1
|
|
|
|
scenario["client_config"]["async_client_threads"] = 1
|
|
|
|
optimization_target = "latency"
|
|
|
|
|
|
|
|
scenario["client_config"]["load_params"] = _load_params(offered_load)
|
|
|
|
|
|
|
|
optimization_channel_arg = {
|
|
|
|
"name": "grpc.optimization_target",
|
|
|
|
"str_value": optimization_target,
|
|
|
|
}
|
|
|
|
scenario["client_config"]["channel_args"].append(optimization_channel_arg)
|
|
|
|
scenario["server_config"]["channel_args"].append(optimization_channel_arg)
|
|
|
|
|
|
|
|
if minimal_stack:
|
|
|
|
_add_channel_arg(scenario["client_config"], "grpc.minimal_stack", 1)
|
|
|
|
_add_channel_arg(scenario["server_config"], "grpc.minimal_stack", 1)
|
|
|
|
|
|
|
|
if messages_per_stream:
|
|
|
|
scenario["client_config"]["messages_per_stream"] = messages_per_stream
|
|
|
|
if client_language:
|
|
|
|
# the CLIENT_LANGUAGE field is recognized by run_performance_tests.py
|
|
|
|
scenario["CLIENT_LANGUAGE"] = client_language
|
|
|
|
if server_language:
|
|
|
|
# the SERVER_LANGUAGE field is recognized by run_performance_tests.py
|
|
|
|
scenario["SERVER_LANGUAGE"] = server_language
|
|
|
|
if categories:
|
|
|
|
scenario["CATEGORIES"] = categories
|
|
|
|
if excluded_poll_engines:
|
|
|
|
# The polling engines for which this scenario is excluded
|
|
|
|
scenario["EXCLUDED_POLL_ENGINES"] = excluded_poll_engines
|
|
|
|
return scenario
|
|
|
|
|
|
|
|
|
|
|
|
class Language(object):
|
|
|
|
@property
|
|
|
|
def safename(self):
|
|
|
|
return str(self)
|
|
|
|
|
|
|
|
|
|
|
|
class CXXLanguage(Language):
|
|
|
|
@property
|
|
|
|
def safename(self):
|
|
|
|
return "cxx"
|
|
|
|
|
|
|
|
def worker_cmdline(self):
|
|
|
|
return ["cmake/build/qps_worker"]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
return 0
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_unary_5000rpcs_1KB_psm",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024,
|
|
|
|
resp_size=1024,
|
|
|
|
outstanding=5000,
|
|
|
|
channels=1,
|
|
|
|
num_clients=1,
|
|
|
|
secure=False,
|
|
|
|
async_server_threads=1,
|
|
|
|
categories=[PSM],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
# TODO(ctiller): add 70% load latency test
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_unary_1channel_100rpcs_1MB",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
unconstrained_client="async",
|
|
|
|
outstanding=100,
|
|
|
|
channels=1,
|
|
|
|
num_clients=1,
|
|
|
|
secure=False,
|
|
|
|
categories=[SWEEP],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_streaming_from_client_1channel_1MB",
|
|
|
|
rpc_type="STREAMING_FROM_CLIENT",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
unconstrained_client="async",
|
|
|
|
outstanding=1,
|
|
|
|
channels=1,
|
|
|
|
num_clients=1,
|
|
|
|
secure=False,
|
|
|
|
categories=[SWEEP],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
# Scenario was added in https://github.com/grpc/grpc/pull/12987, but its purpose is unclear
|
|
|
|
# (beyond excercising some params that other scenarios don't)
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=300,
|
|
|
|
resp_size=50,
|
|
|
|
unconstrained_client="async",
|
|
|
|
outstanding=30000,
|
|
|
|
channels=300,
|
|
|
|
offered_load=37500,
|
|
|
|
secure=False,
|
|
|
|
async_server_threads=16,
|
|
|
|
server_threads_per_cq=1,
|
|
|
|
categories=[SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
for secure in [True, False]:
|
|
|
|
secstr = "secure" if secure else "insecure"
|
|
|
|
smoketest_categories = [SMOKETEST] if secure else []
|
|
|
|
inproc_categories = [INPROC] if not secure else []
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_generic_async_streaming_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
use_generic_payload=True,
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
categories=smoketest_categories
|
|
|
|
+ inproc_categories
|
|
|
|
+ [SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_generic_async_streaming_qps_unconstrained_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=secure,
|
|
|
|
client_threads_per_cq=2,
|
|
|
|
server_threads_per_cq=2,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=smoketest_categories
|
|
|
|
+ inproc_categories
|
|
|
|
+ [SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
for mps in geometric_progression(10, 20, 10):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_generic_async_streaming_qps_unconstrained_%smps_%s"
|
|
|
|
% (mps, secstr),
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=secure,
|
|
|
|
messages_per_stream=mps,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=smoketest_categories
|
|
|
|
+ inproc_categories
|
|
|
|
+ [SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
for mps in geometric_progression(1, 200, math.sqrt(10)):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_generic_async_streaming_qps_unconstrained_%smps_%s"
|
|
|
|
% (mps, secstr),
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=secure,
|
|
|
|
messages_per_stream=mps,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=[SWEEP],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_generic_async_streaming_qps_1channel_1MBmsg_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=secure,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=inproc_categories + [SCALABLE],
|
|
|
|
channels=1,
|
|
|
|
outstanding=100,
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s"
|
|
|
|
% secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
req_size=64 * 1024,
|
|
|
|
resp_size=64 * 1024,
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=secure,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=inproc_categories + [SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_generic_async_streaming_qps_unconstrained_1cq_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async-limited",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=secure,
|
|
|
|
client_threads_per_cq=1000000,
|
|
|
|
server_threads_per_cq=1000000,
|
|
|
|
categories=[SWEEP],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s"
|
|
|
|
% secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async-limited",
|
|
|
|
secure=secure,
|
|
|
|
client_threads_per_cq=1000000,
|
|
|
|
server_threads_per_cq=1000000,
|
|
|
|
categories=inproc_categories + [SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_unary_qps_unconstrained_1cq_%s" % secstr,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async-limited",
|
|
|
|
secure=secure,
|
|
|
|
client_threads_per_cq=1000000,
|
|
|
|
server_threads_per_cq=1000000,
|
|
|
|
categories=inproc_categories + [SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_generic_async_streaming_qps_one_server_core_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async-limited",
|
|
|
|
use_generic_payload=True,
|
|
|
|
async_server_threads=1,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
secure=secure,
|
|
|
|
categories=[SWEEP],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s"
|
|
|
|
% (secstr),
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
secure=secure,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=smoketest_categories
|
|
|
|
+ inproc_categories
|
|
|
|
+ [SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s"
|
|
|
|
% (secstr),
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
channels=1,
|
|
|
|
outstanding=64,
|
|
|
|
req_size=128,
|
|
|
|
resp_size=8 * 1024 * 1024,
|
|
|
|
secure=secure,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=inproc_categories + [SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s"
|
|
|
|
% secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
secure=secure,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=[SWEEP],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_async_unary_ping_pong_%s_1MB" % secstr,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
secure=secure,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=smoketest_categories
|
|
|
|
+ inproc_categories
|
|
|
|
+ [SCALABLE, DASHBOARD],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
for rpc_type in [
|
|
|
|
"unary",
|
|
|
|
"streaming",
|
|
|
|
"streaming_from_client",
|
|
|
|
"streaming_from_server",
|
|
|
|
]:
|
|
|
|
maybe_dashboard = (
|
|
|
|
[DASHBOARD] if rpc_type in ("unary", "streaming") else []
|
|
|
|
)
|
|
|
|
for synchronicity in ["sync", "async"]:
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_%s_%s_ping_pong_%s"
|
|
|
|
% (synchronicity, rpc_type, secstr),
|
|
|
|
rpc_type=rpc_type.upper(),
|
|
|
|
client_type="%s_CLIENT" % synchronicity.upper(),
|
|
|
|
server_type="%s_SERVER" % synchronicity.upper(),
|
|
|
|
async_server_threads=1,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
secure=secure,
|
|
|
|
categories=list(DEFAULT_CATEGORIES) + maybe_dashboard,
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
for size in geometric_progression(
|
|
|
|
1, 1024 * 1024 * 1024 + 1, 8
|
|
|
|
):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_%s_%s_qps_unconstrained_%s_%db"
|
|
|
|
% (synchronicity, rpc_type, secstr, size),
|
|
|
|
rpc_type=rpc_type.upper(),
|
|
|
|
req_size=size,
|
|
|
|
resp_size=size,
|
|
|
|
client_type="%s_CLIENT" % synchronicity.upper(),
|
|
|
|
server_type="%s_SERVER" % synchronicity.upper(),
|
|
|
|
unconstrained_client=synchronicity,
|
|
|
|
secure=secure,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=[SWEEP],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
maybe_scalable = [SCALABLE]
|
|
|
|
if (
|
|
|
|
rpc_type == "streaming_from_server"
|
|
|
|
and synchronicity == "async"
|
|
|
|
and secure
|
|
|
|
):
|
|
|
|
# protobuf_async_streaming_from_server_qps_unconstrained_secure is very flaky
|
|
|
|
# and has extremely high variance so running it isn't really useful.
|
|
|
|
# see b/198275705
|
|
|
|
maybe_scalable = [SWEEP]
|
|
|
|
|
|
|
|
maybe_dashboard = (
|
|
|
|
[DASHBOARD]
|
|
|
|
if rpc_type in ("unary", "streaming")
|
|
|
|
else []
|
|
|
|
)
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_%s_%s_qps_unconstrained_%s"
|
|
|
|
% (synchronicity, rpc_type, secstr),
|
|
|
|
rpc_type=rpc_type.upper(),
|
|
|
|
client_type="%s_CLIENT" % synchronicity.upper(),
|
|
|
|
server_type="%s_SERVER" % synchronicity.upper(),
|
|
|
|
unconstrained_client=synchronicity,
|
|
|
|
secure=secure,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
server_threads_per_cq=2,
|
|
|
|
client_threads_per_cq=2,
|
|
|
|
categories=inproc_categories
|
|
|
|
+ maybe_scalable
|
|
|
|
+ maybe_dashboard,
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
# TODO(vjpai): Re-enable this test. It has a lot of timeouts
|
|
|
|
# and hasn't yet been conclusively identified as a test failure
|
|
|
|
# or race in the library
|
|
|
|
# yield _ping_pong_scenario(
|
|
|
|
# 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr),
|
|
|
|
# rpc_type=rpc_type.upper(),
|
|
|
|
# client_type='%s_CLIENT' % synchronicity.upper(),
|
|
|
|
# server_type='%s_SERVER' % synchronicity.upper(),
|
|
|
|
# unconstrained_client=synchronicity,
|
|
|
|
# secure=secure,
|
|
|
|
# categories=smoketest_categories+[SCALABLE],
|
|
|
|
# warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
# resource_quota_size=500*1024)
|
|
|
|
|
|
|
|
if rpc_type == "streaming":
|
|
|
|
for mps in geometric_progression(10, 20, 10):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s"
|
|
|
|
% (synchronicity, rpc_type, mps, secstr),
|
|
|
|
rpc_type=rpc_type.upper(),
|
|
|
|
client_type="%s_CLIENT" % synchronicity.upper(),
|
|
|
|
server_type="%s_SERVER" % synchronicity.upper(),
|
|
|
|
unconstrained_client=synchronicity,
|
|
|
|
secure=secure,
|
|
|
|
messages_per_stream=mps,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=inproc_categories + [SCALABLE],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
for mps in geometric_progression(1, 200, math.sqrt(10)):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s"
|
|
|
|
% (synchronicity, rpc_type, mps, secstr),
|
|
|
|
rpc_type=rpc_type.upper(),
|
|
|
|
client_type="%s_CLIENT" % synchronicity.upper(),
|
|
|
|
server_type="%s_SERVER" % synchronicity.upper(),
|
|
|
|
unconstrained_client=synchronicity,
|
|
|
|
secure=secure,
|
|
|
|
messages_per_stream=mps,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=[SWEEP],
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
for channels in geometric_progression(
|
|
|
|
1, 20000, math.sqrt(10)
|
|
|
|
):
|
|
|
|
for outstanding in geometric_progression(
|
|
|
|
1, 200000, math.sqrt(10)
|
|
|
|
):
|
|
|
|
if synchronicity == "sync" and outstanding > 1200:
|
|
|
|
continue
|
|
|
|
if outstanding < channels:
|
|
|
|
continue
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding"
|
|
|
|
% (
|
|
|
|
synchronicity,
|
|
|
|
rpc_type,
|
|
|
|
secstr,
|
|
|
|
channels,
|
|
|
|
outstanding,
|
|
|
|
),
|
|
|
|
rpc_type=rpc_type.upper(),
|
|
|
|
client_type="%s_CLIENT" % synchronicity.upper(),
|
|
|
|
server_type="%s_SERVER" % synchronicity.upper(),
|
|
|
|
unconstrained_client=synchronicity,
|
|
|
|
secure=secure,
|
|
|
|
minimal_stack=not secure,
|
|
|
|
categories=[SWEEP],
|
|
|
|
channels=channels,
|
|
|
|
outstanding=outstanding,
|
|
|
|
warmup_seconds=CXX_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "c++"
|
|
|
|
|
|
|
|
|
|
|
|
class CSharpLanguage(Language):
|
|
|
|
"""The legacy Grpc.Core implementation from grpc/grpc."""
|
|
|
|
|
|
|
|
def worker_cmdline(self):
|
|
|
|
return ["tools/run_tests/performance/run_worker_csharp.sh"]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
return 100
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_generic_async_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
use_generic_payload=True,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_generic_async_streaming_ping_pong_insecure_1MB",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=False,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_generic_async_streaming_qps_unconstrained_insecure",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=False,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_protobuf_async_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_protobuf_async_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_protobuf_sync_to_async_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_protobuf_async_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_protobuf_async_streaming_qps_unconstrained",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_to_cpp_protobuf_sync_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=1,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_to_cpp_protobuf_async_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=1,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_to_cpp_protobuf_async_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
server_language="c++",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="sync",
|
|
|
|
server_language="c++",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_to_csharp_protobuf_async_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
client_language="c++",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"csharp_protobuf_async_unary_ping_pong_1MB",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "csharp"
|
|
|
|
|
|
|
|
|
|
|
|
class DotnetLanguage(Language):
|
|
|
|
"""The pure C# implementation from grpc/grpc-dotnet."""
|
|
|
|
|
|
|
|
def worker_cmdline(self):
|
|
|
|
# grpc-dotnet worker is only supported by the new GKE based OSS benchmark
|
|
|
|
# framework, and the worker_cmdline() is only used by run_performance_tests.py
|
|
|
|
return ["grpc_dotnet_not_supported_by_legacy_performance_runner.sh"]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
return 1100
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_generic_async_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
use_generic_payload=True,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_generic_async_streaming_ping_pong_insecure_1MB",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=False,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_generic_async_streaming_qps_unconstrained_insecure",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=False,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_protobuf_async_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_protobuf_async_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_protobuf_sync_to_async_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_protobuf_async_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_protobuf_async_streaming_qps_unconstrained",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_to_cpp_protobuf_sync_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=1,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_to_cpp_protobuf_async_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=1,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_to_cpp_protobuf_async_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
server_language="c++",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="sync",
|
|
|
|
server_language="c++",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"cpp_to_dotnet_protobuf_async_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
client_language="c++",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"dotnet_protobuf_async_unary_ping_pong_1MB",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "dotnet"
|
|
|
|
|
|
|
|
|
|
|
|
class PythonLanguage(Language):
|
|
|
|
def worker_cmdline(self):
|
|
|
|
return ["tools/run_tests/performance/run_worker_python.sh"]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
return 500
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_protobuf_async_unary_5000rpcs_1KB_psm",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024,
|
|
|
|
resp_size=1024,
|
|
|
|
outstanding=5000,
|
|
|
|
channels=1,
|
|
|
|
num_clients=1,
|
|
|
|
secure=False,
|
|
|
|
async_server_threads=1,
|
|
|
|
categories=[PSM],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_generic_sync_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
use_generic_payload=True,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_protobuf_sync_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_protobuf_async_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_protobuf_sync_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_protobuf_sync_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="sync",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_protobuf_sync_streaming_qps_unconstrained",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="sync",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_to_cpp_protobuf_sync_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=0,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_to_cpp_protobuf_sync_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=1,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_protobuf_sync_unary_ping_pong_1MB",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "python"
|
|
|
|
|
|
|
|
|
|
|
|
class PythonAsyncIOLanguage(Language):
|
|
|
|
def worker_cmdline(self):
|
|
|
|
return ["tools/run_tests/performance/run_worker_python_asyncio.sh"]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
return 1200
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_protobuf_async_unary_5000rpcs_1KB_psm",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024,
|
|
|
|
resp_size=1024,
|
|
|
|
outstanding=5000,
|
|
|
|
channels=1,
|
|
|
|
num_clients=1,
|
|
|
|
secure=False,
|
|
|
|
async_server_threads=1,
|
|
|
|
categories=[PSM],
|
|
|
|
)
|
|
|
|
|
|
|
|
for outstanding in [64, 128, 256, 512]:
|
|
|
|
for channels in [1, 4]:
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_protobuf_async_unary_ping_pong_%dx%d_max"
|
|
|
|
% (
|
|
|
|
outstanding,
|
|
|
|
channels,
|
|
|
|
),
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
outstanding=outstanding * channels,
|
|
|
|
channels=channels,
|
|
|
|
client_processes=0,
|
|
|
|
server_processes=0,
|
|
|
|
unconstrained_client="async",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_protobuf_async_unary_ping_pong_%d_1thread"
|
|
|
|
% outstanding,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
outstanding=outstanding,
|
|
|
|
channels=1,
|
|
|
|
client_processes=1,
|
|
|
|
server_processes=1,
|
|
|
|
unconstrained_client="async",
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_generic_async_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
channels=1,
|
|
|
|
client_processes=1,
|
|
|
|
server_processes=1,
|
|
|
|
use_generic_payload=True,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_protobuf_async_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
channels=1,
|
|
|
|
client_processes=1,
|
|
|
|
server_processes=1,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_protobuf_async_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
client_processes=1,
|
|
|
|
server_processes=1,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_protobuf_async_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
channels=1,
|
|
|
|
client_processes=1,
|
|
|
|
server_processes=1,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_protobuf_async_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
channels=1,
|
|
|
|
unconstrained_client="async",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_protobuf_async_streaming_qps_unconstrained",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
channels=1,
|
|
|
|
unconstrained_client="async",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_to_cpp_protobuf_async_unary_ping_pong_1thread",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
channels=1,
|
|
|
|
client_processes=1,
|
|
|
|
unconstrained_client="async",
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_to_cpp_protobuf_async_unary_ping_pong_max",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
channels=1,
|
|
|
|
client_processes=0,
|
|
|
|
server_language="c++",
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_to_cpp_protobuf_sync_streaming_ping_pong_1thread",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
channels=1,
|
|
|
|
client_processes=1,
|
|
|
|
server_processes=1,
|
|
|
|
unconstrained_client="async",
|
|
|
|
server_language="c++",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"python_asyncio_protobuf_async_unary_ping_pong_1MB",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
channels=1,
|
|
|
|
client_processes=1,
|
|
|
|
server_processes=1,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "python_asyncio"
|
|
|
|
|
|
|
|
|
|
|
|
class RubyLanguage(Language):
|
|
|
|
def worker_cmdline(self):
|
|
|
|
return ["tools/run_tests/performance/run_worker_ruby.sh"]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
return 300
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"ruby_protobuf_sync_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"ruby_protobuf_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"ruby_protobuf_sync_unary_qps_unconstrained",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
unconstrained_client="sync",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"ruby_protobuf_sync_streaming_qps_unconstrained",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
unconstrained_client="sync",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"ruby_to_cpp_protobuf_sync_unary_ping_pong",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=1,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"ruby_to_cpp_protobuf_sync_streaming_ping_pong",
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=1,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"ruby_protobuf_unary_ping_pong_1MB",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
req_size=1024 * 1024,
|
|
|
|
resp_size=1024 * 1024,
|
|
|
|
categories=[SMOKETEST, SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "ruby"
|
|
|
|
|
|
|
|
|
|
|
|
class Php7Language(Language):
|
|
|
|
def __init__(self, php7_protobuf_c=False):
|
|
|
|
super().__init__()
|
|
|
|
self.php7_protobuf_c = php7_protobuf_c
|
|
|
|
|
|
|
|
def worker_cmdline(self):
|
|
|
|
if self.php7_protobuf_c:
|
|
|
|
return [
|
|
|
|
"tools/run_tests/performance/run_worker_php.sh",
|
|
|
|
"--use_protobuf_c_extension",
|
|
|
|
]
|
|
|
|
return ["tools/run_tests/performance/run_worker_php.sh"]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
if self.php7_protobuf_c:
|
|
|
|
return 900
|
|
|
|
return 800
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
php7_extension_mode = "php7_protobuf_php_extension"
|
|
|
|
if self.php7_protobuf_c:
|
|
|
|
php7_extension_mode = "php7_protobuf_c_extension"
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"%s_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm"
|
|
|
|
% php7_extension_mode,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
req_size=1024,
|
|
|
|
resp_size=1024,
|
|
|
|
outstanding=5000,
|
|
|
|
channels=1,
|
|
|
|
num_clients=1,
|
|
|
|
secure=False,
|
|
|
|
async_server_threads=1,
|
|
|
|
categories=[PSM],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"%s_to_cpp_protobuf_sync_unary_ping_pong" % php7_extension_mode,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=1,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"%s_to_cpp_protobuf_sync_streaming_ping_pong" % php7_extension_mode,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
async_server_threads=1,
|
|
|
|
)
|
|
|
|
|
|
|
|
# TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs
|
|
|
|
# better than async_server_threads=0/CPU usage 490%.
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"%s_to_cpp_protobuf_sync_unary_qps_unconstrained"
|
|
|
|
% php7_extension_mode,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
outstanding=1,
|
|
|
|
async_server_threads=1,
|
|
|
|
unconstrained_client="sync",
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"%s_to_cpp_protobuf_sync_streaming_qps_unconstrained"
|
|
|
|
% php7_extension_mode,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
server_language="c++",
|
|
|
|
outstanding=1,
|
|
|
|
async_server_threads=1,
|
|
|
|
unconstrained_client="sync",
|
|
|
|
)
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
if self.php7_protobuf_c:
|
|
|
|
return "php7_protobuf_c"
|
|
|
|
return "php7"
|
|
|
|
|
|
|
|
|
|
|
|
class JavaLanguage(Language):
|
|
|
|
def worker_cmdline(self):
|
|
|
|
return ["tools/run_tests/performance/run_worker_java.sh"]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
return 400
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"java_protobuf_async_unary_5000rpcs_1KB_psm",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024,
|
|
|
|
resp_size=1024,
|
|
|
|
outstanding=5000,
|
|
|
|
channels=1,
|
|
|
|
num_clients=1,
|
|
|
|
secure=False,
|
|
|
|
async_server_threads=1,
|
|
|
|
warmup_seconds=JAVA_WARMUP_SECONDS,
|
|
|
|
categories=[PSM],
|
|
|
|
)
|
|
|
|
|
|
|
|
for secure in [True, False]:
|
|
|
|
secstr = "secure" if secure else "insecure"
|
|
|
|
smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"java_generic_async_streaming_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
use_generic_payload=True,
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
warmup_seconds=JAVA_WARMUP_SECONDS,
|
|
|
|
categories=smoketest_categories,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"java_protobuf_async_streaming_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
warmup_seconds=JAVA_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"java_protobuf_async_unary_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
warmup_seconds=JAVA_WARMUP_SECONDS,
|
|
|
|
categories=smoketest_categories,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"java_protobuf_unary_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
warmup_seconds=JAVA_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"java_protobuf_async_unary_qps_unconstrained_%s" % secstr,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
secure=secure,
|
|
|
|
warmup_seconds=JAVA_WARMUP_SECONDS,
|
|
|
|
categories=smoketest_categories + [SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"java_protobuf_async_streaming_qps_unconstrained_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
secure=secure,
|
|
|
|
warmup_seconds=JAVA_WARMUP_SECONDS,
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"java_generic_async_streaming_qps_unconstrained_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=secure,
|
|
|
|
warmup_seconds=JAVA_WARMUP_SECONDS,
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"java_generic_async_streaming_qps_one_server_core_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async-limited",
|
|
|
|
use_generic_payload=True,
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
warmup_seconds=JAVA_WARMUP_SECONDS,
|
|
|
|
)
|
|
|
|
|
|
|
|
# TODO(jtattermusch): add scenarios java vs C++
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "java"
|
|
|
|
|
|
|
|
|
|
|
|
class GoLanguage(Language):
|
|
|
|
def worker_cmdline(self):
|
|
|
|
return ["tools/run_tests/performance/run_worker_go.sh"]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
return 600
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"go_protobuf_async_unary_5000rpcs_1KB_psm",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024,
|
|
|
|
resp_size=1024,
|
|
|
|
outstanding=5000,
|
|
|
|
channels=1,
|
|
|
|
num_clients=1,
|
|
|
|
secure=False,
|
|
|
|
async_server_threads=1,
|
|
|
|
categories=[PSM],
|
|
|
|
)
|
|
|
|
|
|
|
|
for secure in [True, False]:
|
|
|
|
secstr = "secure" if secure else "insecure"
|
|
|
|
smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
|
|
|
|
|
|
|
|
# ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
|
|
|
|
# but that's mostly because of lack of better name of the enum value.
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"go_generic_sync_streaming_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
use_generic_payload=True,
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
categories=smoketest_categories,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"go_protobuf_sync_streaming_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"go_protobuf_sync_unary_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
categories=smoketest_categories,
|
|
|
|
)
|
|
|
|
|
|
|
|
# unconstrained_client='async' is intended (client uses goroutines)
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"go_protobuf_sync_unary_qps_unconstrained_%s" % secstr,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
secure=secure,
|
|
|
|
categories=smoketest_categories + [SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
# unconstrained_client='async' is intended (client uses goroutines)
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"go_protobuf_sync_streaming_qps_unconstrained_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="SYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
secure=secure,
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
# unconstrained_client='async' is intended (client uses goroutines)
|
|
|
|
# ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
|
|
|
|
# but that's mostly because of lack of better name of the enum value.
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"go_generic_sync_streaming_qps_unconstrained_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="SYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=secure,
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
# TODO(jtattermusch): add scenarios go vs C++
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "go"
|
|
|
|
|
|
|
|
|
|
|
|
class NodeLanguage(Language):
|
|
|
|
def worker_cmdline(self):
|
|
|
|
fixture = "js_js"
|
|
|
|
return [
|
|
|
|
"tools/run_tests/performance/run_worker_node.sh",
|
|
|
|
fixture,
|
|
|
|
"--benchmark_impl=grpc",
|
|
|
|
]
|
|
|
|
|
|
|
|
def worker_port_offset(self):
|
|
|
|
return 1100
|
|
|
|
|
|
|
|
def scenarios(self):
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"node_to_node_protobuf_async_unary_5000rpcs_1KB_psm",
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
req_size=1024,
|
|
|
|
resp_size=1024,
|
|
|
|
outstanding=5000,
|
|
|
|
channels=1,
|
|
|
|
num_clients=1,
|
|
|
|
secure=False,
|
|
|
|
async_server_threads=1,
|
|
|
|
categories=[PSM],
|
|
|
|
)
|
|
|
|
|
|
|
|
for secure in [True, False]:
|
|
|
|
secstr = "secure" if secure else "insecure"
|
|
|
|
smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"node_to_node_generic_async_streaming_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
use_generic_payload=True,
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
categories=smoketest_categories,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"node_to_node_protobuf_async_streaming_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"node_to_node_protobuf_async_unary_ping_pong_%s" % secstr,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
async_server_threads=1,
|
|
|
|
secure=secure,
|
|
|
|
categories=smoketest_categories,
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"node_to_node_protobuf_async_unary_qps_unconstrained_%s"
|
|
|
|
% secstr,
|
|
|
|
rpc_type="UNARY",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
secure=secure,
|
|
|
|
categories=smoketest_categories + [SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"node_to_node_protobuf_async_streaming_qps_unconstrained_%s"
|
|
|
|
% secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
secure=secure,
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
yield _ping_pong_scenario(
|
|
|
|
"node_to_node_generic_async_streaming_qps_unconstrained_%s"
|
|
|
|
% secstr,
|
|
|
|
rpc_type="STREAMING",
|
|
|
|
client_type="ASYNC_CLIENT",
|
|
|
|
server_type="ASYNC_GENERIC_SERVER",
|
|
|
|
unconstrained_client="async",
|
|
|
|
use_generic_payload=True,
|
|
|
|
secure=secure,
|
|
|
|
categories=[SCALABLE],
|
|
|
|
)
|
|
|
|
|
|
|
|
# TODO(murgatroid99): add scenarios node vs C++
|
|
|
|
|
|
|
|
def __str__(self):
|
|
|
|
return "node"
|
|
|
|
|
|
|
|
|
|
|
|
LANGUAGES = {
|
|
|
|
"c++": CXXLanguage(),
|
|
|
|
"csharp": CSharpLanguage(),
|
|
|
|
"dotnet": DotnetLanguage(),
|
|
|
|
"ruby": RubyLanguage(),
|
|
|
|
"php7": Php7Language(),
|
|
|
|
"php7_protobuf_c": Php7Language(php7_protobuf_c=True),
|
|
|
|
"java": JavaLanguage(),
|
|
|
|
"python": PythonLanguage(),
|
|
|
|
"python_asyncio": PythonAsyncIOLanguage(),
|
|
|
|
"go": GoLanguage(),
|
|
|
|
"node": NodeLanguage(), # 'node' means 'node_purejs'.
|
|
|
|
}
|