mirror of https://github.com/grpc/grpc.git
Run example benchmarks v2 test in continuous build. (#25976)
This commit includes the following changes: 1. A new load test template generator (loadtest_template.py) is added. The template generator combines existing configurations or templates for several languages into a single template that can be used to generate configurations for different languages or combinations of languages. 2. A basic template generated from the example tests in grpc/test-infra (loadtest_template_basic_all_languages.yaml) is added. 3. The load test config generator is updated to use the combined template. 4. An example run consisting of a single test (generated from the combined template) is added and set up to run continuously.pull/26036/head
Paulo Castello da Costa
4 years ago
committed by
GitHub
parent
57cb063fb7
commit
bb418da2b5
8 changed files with 897 additions and 152 deletions
@ -0,0 +1,203 @@ |
||||
#!/usr/bin/env python3 |
||||
# Copyright 2021 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# This script generates a load test configuration template from a collection of |
||||
# load test configurations. |
||||
# |
||||
# Configuration templates contain client and server configurations for multiple |
||||
# languages, and may contain template substitution keys. These templates are |
||||
# used to generate load test configurations by selecting clients and servers for |
||||
# the required languages. The source files for template generation may be load |
||||
# test configurations or load test configuration templates. Load test |
||||
# configuration generation is performed by loadtest_config.py. See documentation |
||||
# below: |
||||
# https://github.com/grpc/grpc/blob/master/tools/run_tests/performance/README.md |
||||
|
||||
import argparse |
||||
import sys |
||||
|
||||
from typing import Any, Dict, Iterable, Mapping, Type |
||||
|
||||
import yaml |
||||
|
||||
import loadtest_config |
||||
|
||||
TEMPLATE_FILE_HEADER_COMMENT = """ |
||||
# Template generated from load test configurations by loadtest_template.py. |
||||
# |
||||
# Configuration templates contain client and server configurations for multiple |
||||
# languages, and may contain template substitution keys. These templates are |
||||
# used to generate load test configurations by selecting clients and servers for |
||||
# the required languages. The source files for template generation may be load |
||||
# test configurations or load test configuration templates. Load test |
||||
# configuration generation is performed by loadtest_config.py. See documentation |
||||
# below: |
||||
# https://github.com/grpc/grpc/blob/master/tools/run_tests/performance/README.md |
||||
""" |
||||
|
||||
|
||||
def loadtest_template( |
||||
input_file_names: Iterable[str], |
||||
metadata: Mapping[str, Any], |
||||
inject_client_pool: bool, |
||||
inject_server_pool: bool, |
||||
inject_big_query_table: bool, |
||||
inject_timeout_seconds: bool, |
||||
inject_ttl_seconds: bool) -> Dict[str, Any]: # yapf: disable |
||||
"""Generates the load test template.""" |
||||
clients = list() |
||||
servers = list() |
||||
spec = dict() |
||||
client_languages = set() |
||||
server_languages = set() |
||||
template = { |
||||
'apiVersion': 'e2etest.grpc.io/v1', |
||||
'kind': 'LoadTest', |
||||
'metadata': metadata, |
||||
} |
||||
for input_file_name in input_file_names: |
||||
with open(input_file_name) as f: |
||||
input_config = yaml.safe_load(f.read()) |
||||
|
||||
if input_config.get('apiVersion') != template['apiVersion']: |
||||
raise ValueError('Unexpected api version in file {}: {}'.format( |
||||
input_file_name, input_config.get('apiVersion'))) |
||||
if input_config.get('kind') != template['kind']: |
||||
raise ValueError('Unexpected kind in file {}: {}'.format( |
||||
input_file_name, input_config.get('kind'))) |
||||
|
||||
for client in input_config['spec']['clients']: |
||||
if client['language'] in client_languages: |
||||
continue |
||||
if inject_client_pool: |
||||
client['pool'] = '${client_pool}' |
||||
clients.append(client) |
||||
client_languages.add(client['language']) |
||||
|
||||
for server in input_config['spec']['servers']: |
||||
if server['language'] in server_languages: |
||||
continue |
||||
if inject_server_pool: |
||||
server['pool'] = '${server_pool}' |
||||
servers.append(server) |
||||
server_languages.add(server['language']) |
||||
|
||||
input_spec = input_config['spec'] |
||||
del input_spec['clients'] |
||||
del input_spec['servers'] |
||||
del input_spec['scenariosJSON'] |
||||
spec.update(input_config['spec']) |
||||
|
||||
clients.sort(key=lambda x: x['language']) |
||||
servers.sort(key=lambda x: x['language']) |
||||
|
||||
spec.update({ |
||||
'clients': clients, |
||||
'servers': servers, |
||||
}) |
||||
|
||||
if inject_big_query_table: |
||||
spec['big_query_table'] = '${big_query_table}' |
||||
if inject_timeout_seconds: |
||||
spec['timeoutSeconds'] = '${timeout_seconds}' |
||||
if inject_ttl_seconds: |
||||
spec['ttlSeconds'] = '${ttl_seconds}' |
||||
|
||||
template['spec'] = spec |
||||
|
||||
return template |
||||
|
||||
|
||||
def template_dumper(header_comment: str) -> Type[yaml.Dumper]: |
||||
"""Returns a custom dumper to dump templates in the expected format.""" |
||||
|
||||
class TemplateDumper(yaml.Dumper): |
||||
|
||||
def expect_stream_start(self): |
||||
super().expect_stream_start() |
||||
if isinstance(self.event, yaml.StreamStartEvent): |
||||
self.write_indent() |
||||
self.write_indicator(header_comment, need_whitespace=False) |
||||
|
||||
return TemplateDumper |
||||
|
||||
|
||||
def main() -> None: |
||||
argp = argparse.ArgumentParser( |
||||
description='Creates a load test config generator template.', |
||||
fromfile_prefix_chars='@') |
||||
argp.add_argument('-i', |
||||
'--inputs', |
||||
action='extend', |
||||
nargs='+', |
||||
type=str, |
||||
help='Input files.') |
||||
argp.add_argument('-o', |
||||
'--output', |
||||
type=str, |
||||
help='Output file. Outputs to stdout if not set.') |
||||
argp.add_argument( |
||||
'--inject_client_pool', |
||||
action='store_true', |
||||
help='Set spec.client(s).pool values to \'${client_pool}\'.') |
||||
argp.add_argument( |
||||
'--inject_server_pool', |
||||
action='store_true', |
||||
help='Set spec.server(s).pool values to \'${server_pool}\'.') |
||||
argp.add_argument('--inject_big_query_table', |
||||
action='store_true', |
||||
help='Set spec.bigQueryTable to \'${big_query_table}\'.') |
||||
argp.add_argument('--inject_timeout_seconds', |
||||
action='store_true', |
||||
help='Set spec.timeoutSeconds to \'${timeout_seconds}\'.') |
||||
argp.add_argument('--inject_ttl_seconds', |
||||
action='store_true', |
||||
help='Set timeout ') |
||||
argp.add_argument('-n', |
||||
'--name', |
||||
default='', |
||||
type=str, |
||||
help='metadata.name.') |
||||
argp.add_argument('-a', |
||||
'--annotation', |
||||
action='append', |
||||
type=str, |
||||
help='metadata.annotation(s), in the form key=value.', |
||||
dest='annotations') |
||||
args = argp.parse_args() |
||||
|
||||
annotations = loadtest_config.parse_key_value_args(args.annotations) |
||||
|
||||
metadata = {'name': args.name} |
||||
if annotations: |
||||
metadata['annotations'] = annotations |
||||
|
||||
template = loadtest_template( |
||||
input_file_names=args.inputs, |
||||
metadata=metadata, |
||||
inject_client_pool=args.inject_client_pool, |
||||
inject_server_pool=args.inject_server_pool, |
||||
inject_big_query_table=args.inject_big_query_table, |
||||
inject_timeout_seconds=args.inject_timeout_seconds, |
||||
inject_ttl_seconds=args.inject_ttl_seconds) |
||||
|
||||
with open(args.output, 'w') if args.output else sys.stdout as f: |
||||
yaml.dump(template, |
||||
stream=f, |
||||
Dumper=template_dumper(TEMPLATE_FILE_HEADER_COMMENT.strip())) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
main() |
||||
@ -0,0 +1,258 @@ |
||||
# Template generated from load test configurations by loadtest_template.py. |
||||
# |
||||
# Configuration templates contain client and server configurations for multiple |
||||
# languages, and may contain template substitution keys. These templates are |
||||
# used to generate load test configurations by selecting clients and servers for |
||||
# the required languages. The source files for template generation may be load |
||||
# test configurations or load test configuration templates. Load test |
||||
# configuration generation is performed by loadtest_config.py. See documentation |
||||
# below: |
||||
# https://github.com/grpc/grpc/blob/master/tools/run_tests/performance/README.md |
||||
apiVersion: e2etest.grpc.io/v1 |
||||
kind: LoadTest |
||||
metadata: |
||||
name: basic_all_languages |
||||
spec: |
||||
big_query_table: ${big_query_table} |
||||
clients: |
||||
- build: |
||||
command: |
||||
- bash |
||||
- /build_scripts/build_qps_worker.sh |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc.git |
||||
language: csharp |
||||
pool: ${client_pool} |
||||
run: |
||||
args: |
||||
- exec |
||||
- qps_worker/Grpc.IntegrationTesting.QpsWorker.dll |
||||
command: |
||||
- dotnet |
||||
- build: |
||||
args: |
||||
- build |
||||
- //test/cpp/qps:qps_worker |
||||
command: |
||||
- bazel |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc.git |
||||
language: cxx |
||||
pool: ${client_pool} |
||||
run: |
||||
command: |
||||
- bazel-bin/test/cpp/qps/qps_worker |
||||
- build: |
||||
args: |
||||
- build |
||||
- -o |
||||
- /src/workspace/bin/worker |
||||
- ./benchmark/worker |
||||
command: |
||||
- go |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc-go.git |
||||
language: go |
||||
pool: ${client_pool} |
||||
run: |
||||
command: |
||||
- /src/workspace/bin/worker |
||||
- build: |
||||
args: |
||||
- -PskipAndroid=true |
||||
- -PskipCodegen=true |
||||
- :grpc-benchmarks:installDist |
||||
command: |
||||
- gradle |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc-java.git |
||||
language: java |
||||
pool: ${client_pool} |
||||
run: |
||||
command: |
||||
- benchmarks/build/install/grpc-benchmarks/bin/benchmark_worker |
||||
- build: |
||||
command: |
||||
- bash |
||||
- /build_scripts/build_qps_worker.sh |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc-node.git |
||||
language: node |
||||
pool: ${client_pool} |
||||
run: |
||||
args: |
||||
- -r |
||||
- ./test/fixtures/native_native.js |
||||
- test/performance/worker.js |
||||
- --benchmark_impl=grpc |
||||
command: |
||||
- node |
||||
- build: |
||||
command: |
||||
- bash |
||||
- /build_scripts/build_qps_worker.sh |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc.git |
||||
language: php |
||||
pool: ${client_pool} |
||||
run: |
||||
command: |
||||
- bash |
||||
- /run_scripts/run_worker.sh |
||||
- build: |
||||
args: |
||||
- build |
||||
- //src/python/grpcio_tests/tests/qps:qps_worker |
||||
command: |
||||
- bazel |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc.git |
||||
language: python |
||||
pool: ${client_pool} |
||||
run: |
||||
command: |
||||
- bazel-bin/src/python/grpcio_tests/tests/qps/qps_worker |
||||
- build: |
||||
command: |
||||
- bash |
||||
- /build_scripts/build_qps_worker.sh |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc |
||||
language: ruby |
||||
pool: ${client_pool} |
||||
run: |
||||
args: |
||||
- src/ruby/qps/worker.rb |
||||
command: |
||||
- ruby |
||||
servers: |
||||
- build: |
||||
command: |
||||
- bash |
||||
- /build_scripts/build_qps_worker.sh |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc.git |
||||
language: csharp |
||||
pool: ${server_pool} |
||||
run: |
||||
args: |
||||
- exec |
||||
- qps_worker/Grpc.IntegrationTesting.QpsWorker.dll |
||||
command: |
||||
- dotnet |
||||
- build: |
||||
args: |
||||
- build |
||||
- //test/cpp/qps:qps_worker |
||||
command: |
||||
- bazel |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc.git |
||||
language: cxx |
||||
pool: ${server_pool} |
||||
run: |
||||
args: |
||||
- --server_port=10010 |
||||
command: |
||||
- bazel-bin/test/cpp/qps/qps_worker |
||||
- build: |
||||
args: |
||||
- build |
||||
- -o |
||||
- /src/workspace/bin/worker |
||||
- ./benchmark/worker |
||||
command: |
||||
- go |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc-go.git |
||||
language: go |
||||
pool: ${server_pool} |
||||
run: |
||||
command: |
||||
- /src/workspace/bin/worker |
||||
- build: |
||||
args: |
||||
- -PskipAndroid=true |
||||
- -PskipCodegen=true |
||||
- :grpc-benchmarks:installDist |
||||
command: |
||||
- gradle |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc-java.git |
||||
language: java |
||||
pool: ${server_pool} |
||||
run: |
||||
command: |
||||
- benchmarks/build/install/grpc-benchmarks/bin/benchmark_worker |
||||
- build: |
||||
command: |
||||
- bash |
||||
- /build_scripts/build_qps_worker.sh |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc-node.git |
||||
language: node |
||||
pool: ${server_pool} |
||||
run: |
||||
args: |
||||
- -r |
||||
- ./test/fixtures/native_native.js |
||||
- test/performance/worker.js |
||||
- --benchmark_impl=grpc |
||||
command: |
||||
- node |
||||
- build: |
||||
command: |
||||
- bash |
||||
- /build_scripts/build_qps_worker.sh |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc.git |
||||
language: php |
||||
pool: ${server_pool} |
||||
run: |
||||
command: |
||||
- bash |
||||
- /run_scripts/run_worker.sh |
||||
- build: |
||||
args: |
||||
- build |
||||
- //src/python/grpcio_tests/tests/qps:qps_worker |
||||
command: |
||||
- bazel |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc.git |
||||
language: python |
||||
pool: ${server_pool} |
||||
run: |
||||
command: |
||||
- bazel-bin/src/python/grpcio_tests/tests/qps/qps_worker |
||||
- build: |
||||
command: |
||||
- bash |
||||
- /build_scripts/build_qps_worker.sh |
||||
clone: |
||||
gitRef: master |
||||
repo: https://github.com/grpc/grpc |
||||
language: ruby |
||||
pool: ${server_pool} |
||||
run: |
||||
args: |
||||
- src/ruby/qps/worker.rb |
||||
command: |
||||
- ruby |
||||
timeoutSeconds: ${timeout_seconds} |
||||
ttlSeconds: 86400 |
||||
Loading…
Reference in new issue