Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add scenario generation for PSM tests. (#29336)

Browse Source 
This commit adds the PSM base scenario to each language in
scenario_config.py allowing per language generation of PSM
base scenarios. The PSM base scenarios will be further updated
in loadtest_config.py for number of client channels and the
number of async_server_thread. After update of the two
aforementioned fields, scenarios varying offered_load will
be generated from the base scenarios.

loadtest_config.py is updated to take number of client channels,
the number of async_servers and a list of targeted offered_load.
pull/29460/head
Wanlin Du 3 years ago committed by GitHub
parent 9f65dfed5b
commit 2b42b24a94
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 225 additions and 15 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 112
      tools/run_tests/performance/loadtest_config.py
  2. 128
      tools/run_tests/performance/scenario_config.py

112
tools/run_tests/performance/loadtest_config.py
Unescape View File

@ -30,7 +30,7 @@ import json
import os
import string
import sys
from typing import Any, Dict, Iterable, Mapping, Optional, Type
from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Type
import yaml
@ -118,17 +118,79 @@ def gen_run_indices(runs_per_test: int) -> Iterable[str]:
yield index_fmt.format(i)
def scenario_name(base_name: str, client_channels: Optional[int],
server_threads: Optional[int], offered_load: Optional[int]):
"""Constructs scenario name from base name and modifiers."""
elements = [base_name]
if client_channels:
elements.append('{:d}channels'.format(client_channels))
if server_threads:
elements.append('{:d}threads'.format(server_threads))
if offered_load:
elements.append('{:d}offered_load'.format(offered_load))
return '_'.join(elements)
def scenario_transform_function(
client_channels: Optional[int], server_threads: Optional[int],
offered_loads: Optional[Iterable[int]]
) -> Optional[Callable[[Iterable[Mapping[str, Any]]], Iterable[Mapping[str,
Any]]]]:
"""Returns a transform to be applied to a list of scenarios."""
if not any((client_channels, server_threads, len(offered_loads))):
return lambda s: s
def _transform(
scenarios: Iterable[Mapping[str,
Any]]) -> Iterable[Mapping[str, Any]]:
"""Transforms scenarios by inserting num of client channels, number of async_server_threads and offered_load."""
for base_scenario in scenarios:
base_name = base_scenario['name']
if client_channels:
base_scenario['client_config'][
'client_channels'] = client_channels
if server_threads:
base_scenario['server_config'][
'async_server_threads'] = server_threads
if not offered_loads:
base_scenario['name'] = scenario_name(base_name,
client_channels,
server_threads, 0)
yield base_scenario
return
for offered_load in offered_loads:
scenario = copy.deepcopy(base_scenario)
scenario['client_config']['load_params'] = {
'poisson': {
'offered_load': offered_load
}
}
scenario['name'] = scenario_name(base_name, client_channels,
server_threads, offered_load)
yield scenario
return _transform
def gen_loadtest_configs(
base_config: Mapping[str, Any],
base_config_clients: Iterable[Mapping[str, Any]],
base_config_servers: Iterable[Mapping[str, Any]],
scenario_name_regex: str,
language_config: scenario_config_exporter.LanguageConfig,
loadtest_name_prefix: str,
uniquifier_elements: Iterable[str],
annotations: Mapping[str, str],
instances_per_client: int = 1,
runs_per_test: int = 1) -> Iterable[Dict[str, Any]]:
base_config: Mapping[str, Any],
base_config_clients: Iterable[Mapping[str, Any]],
base_config_servers: Iterable[Mapping[str, Any]],
scenario_name_regex: str,
language_config: scenario_config_exporter.LanguageConfig,
loadtest_name_prefix: str,
uniquifier_elements: Iterable[str],
annotations: Mapping[str, str],
instances_per_client: int = 1,
runs_per_test: int = 1,
scenario_transform: Callable[[Iterable[Mapping[str, Any]]],
List[Dict[str, Any]]] = lambda s: s
) -> Iterable[Dict[str, Any]]:
"""Generates LoadTest configurations for a given language config.
The LoadTest configurations are generated as YAML objects.
@ -142,8 +204,10 @@ def gen_loadtest_configs(
category=language_config.category,
client_language=language_config.client_language,
server_language=language_config.server_language)
scenarios = scenario_config_exporter.gen_scenarios(language_config.language,
scenario_filter)
scenarios = scenario_transform(
scenario_config_exporter.gen_scenarios(language_config.language,
scenario_filter))
for scenario in scenarios:
for run_index in gen_run_indices(runs_per_test):
@ -349,7 +413,7 @@ def main() -> None:
help='Regex to select scenarios to run.')
argp.add_argument(
'--category',
choices=['all', 'inproc', 'scalable', 'smoketest', 'sweep'],
choices=['all', 'inproc', 'scalable', 'smoketest', 'sweep', 'psm'],
default='all',
help='Select a category of tests to run.')
argp.add_argument(
@ -378,6 +442,19 @@ def main() -> None:
'--output',
type=str,
help='Output file name. Output to stdout if not set.')
argp.add_argument('--client_channels',
type=int,
help='Number of client channels.')
argp.add_argument('--server_threads',
type=int,
help='Number of async server threads.')
argp.add_argument(
'--offered_loads',
nargs="*",
type=int,
default=[],
help='A list of QPS values at which each load test scenario will be run.'
)
args = argp.parse_args()
if args.instances_per_client < 1:
@ -404,6 +481,10 @@ def main() -> None:
annotations = parse_key_value_args(args.annotations)
transform = scenario_transform_function(args.client_channels,
args.server_threads,
args.offered_loads)
with open(args.template) as f:
base_config = yaml.safe_load(
string.Template(f.read()).substitute(substitutions))
@ -436,7 +517,8 @@ def main() -> None:
uniquifier_elements=uniquifier_elements,
annotations=annotations,
instances_per_client=args.instances_per_client,
runs_per_test=args.runs_per_test))
runs_per_test=args.runs_per_test,
scenario_transform=transform))
configs = (config for config in itertools.chain(*config_generators))
with open(args.output, 'w') if args.output else sys.stdout as f:

128
tools/run_tests/performance/scenario_config.py
Unescape View File

@ -24,6 +24,7 @@ SMOKETEST = 'smoketest'
SCALABLE = 'scalable'
INPROC = 'inproc'
SWEEP = 'sweep'
PSM = 'psm'
DEFAULT_CATEGORIES = (SCALABLE, SMOKETEST)
SECURE_SECARGS = {
@ -254,6 +255,19 @@ class CXXLanguage(Language):
return 0
def scenarios(self):
yield _ping_pong_scenario('cpp_protobuf_async_unary_5000rpcs_1KB_psm',
rpc_type='UNARY',
client_type='ASYNC_CLIENT',
server_type='ASYNC_SERVER',
req_size=1024,
resp_size=1024,
outstanding=5000,
channels=1,
num_clients=1,
secure=False,
async_server_threads=1,
categories=[PSM])
# TODO(ctiller): add 70% load latency test
yield _ping_pong_scenario(
'cpp_protobuf_async_unary_1channel_100rpcs_1MB',
@ -611,6 +625,20 @@ class CSharpLanguage(Language):
return 100
def scenarios(self):
yield _ping_pong_scenario(
'csharp_protobuf_async_unary_5000rpcs_1KB_psm',
rpc_type='UNARY',
client_type='ASYNC_CLIENT',
server_type='ASYNC_SERVER',
req_size=1024,
resp_size=1024,
outstanding=5000,
channels=1,
num_clients=1,
secure=False,
async_server_threads=1,
categories=[PSM])
yield _ping_pong_scenario('csharp_generic_async_streaming_ping_pong',
rpc_type='STREAMING',
client_type='ASYNC_CLIENT',
@ -864,6 +892,20 @@ class PythonLanguage(Language):
return 500
def scenarios(self):
yield _ping_pong_scenario(
'python_protobuf_async_unary_5000rpcs_1KB_psm',
rpc_type='UNARY',
client_type='ASYNC_CLIENT',
server_type='ASYNC_SERVER',
req_size=1024,
resp_size=1024,
outstanding=5000,
channels=1,
num_clients=1,
secure=False,
async_server_threads=1,
categories=[PSM])
yield _ping_pong_scenario('python_generic_sync_streaming_ping_pong',
rpc_type='STREAMING',
client_type='SYNC_CLIENT',
@ -938,6 +980,20 @@ class PythonAsyncIOLanguage(Language):
return 1200
def scenarios(self):
yield _ping_pong_scenario(
'python_asyncio_protobuf_async_unary_5000rpcs_1KB_psm',
rpc_type='UNARY',
client_type='ASYNC_CLIENT',
server_type='ASYNC_SERVER',
req_size=1024,
resp_size=1024,
outstanding=5000,
channels=1,
num_clients=1,
secure=False,
async_server_threads=1,
categories=[PSM])
for outstanding in [64, 128, 256, 512]:
for channels in [1, 4]:
yield _ping_pong_scenario(
@ -1083,6 +1139,19 @@ class RubyLanguage(Language):
return 300
def scenarios(self):
yield _ping_pong_scenario('ruby_protobuf_async_unary_5000rpcs_1KB_psm',
rpc_type='UNARY',
client_type='ASYNC_CLIENT',
server_type='ASYNC_SERVER',
req_size=1024,
resp_size=1024,
outstanding=5000,
channels=1,
num_clients=1,
secure=False,
async_server_threads=1,
categories=[PSM])
yield _ping_pong_scenario('ruby_protobuf_sync_streaming_ping_pong',
rpc_type='STREAMING',
client_type='SYNC_CLIENT',
@ -1159,6 +1228,22 @@ class Php7Language(Language):
if self.php7_protobuf_c:
php7_extension_mode = 'php7_protobuf_c_extension'
yield _ping_pong_scenario(
'%s_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm' %
php7_extension_mode,
rpc_type='UNARY',
client_type='ASYNC_CLIENT',
server_type='ASYNC_SERVER',
server_language='c++',
req_size=1024,
resp_size=1024,
outstanding=5000,
channels=1,
num_clients=1,
secure=False,
async_server_threads=1,
categories=[PSM])
yield _ping_pong_scenario('%s_to_cpp_protobuf_sync_unary_ping_pong' %
php7_extension_mode,
rpc_type='UNARY',
@ -1214,6 +1299,19 @@ class JavaLanguage(Language):
return 400
def scenarios(self):
yield _ping_pong_scenario('java_protobuf_async_unary_5000rpcs_1KB_psm',
rpc_type='UNARY',
client_type='ASYNC_CLIENT',
server_type='ASYNC_SERVER',
req_size=1024,
resp_size=1024,
outstanding=5000,
channels=1,
num_clients=1,
secure=False,
async_server_threads=1,
categories=[PSM])
for secure in [True, False]:
secstr = 'secure' if secure else 'insecure'
smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
@ -1314,6 +1412,19 @@ class GoLanguage(Language):
return 600
def scenarios(self):
yield _ping_pong_scenario('go_protobuf_async_unary_5000rpcs_1KB_psm',
rpc_type='UNARY',
client_type='ASYNC_CLIENT',
server_type='ASYNC_SERVER',
req_size=1024,
resp_size=1024,
outstanding=5000,
channels=1,
num_clients=1,
secure=False,
async_server_threads=1,
categories=[PSM])
for secure in [True, False]:
secstr = 'secure' if secure else 'insecure'
smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
@ -1406,6 +1517,23 @@ class NodeLanguage(Language):
def scenarios(self):
node_implementation = 'node_purejs' if self.node_purejs else 'node'
yield _ping_pong_scenario(
'%s_to_node_protobuf_async_unary_5000rpcs_1KB_psm' %
(node_implementation),
rpc_type='UNARY',
client_type='ASYNC_CLIENT',
server_type='ASYNC_SERVER',
server_language='node',
req_size=1024,
resp_size=1024,
outstanding=5000,
channels=1,
num_clients=1,
secure=False,
async_server_threads=1,
categories=[PSM])
for secure in [True, False]:
secstr = 'secure' if secure else 'insecure'
smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]

Write Preview
Loading…
Cancel
Save