Chiebot-Mirror
/
grpc
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #22444 from ericgribkoff/separate_test_cases

Browse Source 
Split logs by test case and run client separately for each test
pull/22461/head
Eric Gribkoff 5 years ago committed by GitHub
parent 4e79cda71c cbdceeea34
commit 720339c329
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 141 additions and 67 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      tools/internal_ci/linux/grpc_xds.cfg
  2. 26
      tools/internal_ci/linux/grpc_xds.sh
  3. 8
      tools/internal_ci/linux/grpc_xds_python.cfg
  4. 6
      tools/run_tests/python_utils/report_utils.py
  5. 160
      tools/run_tests/run_xds_tests.py

8
tools/internal_ci/linux/grpc_xds.cfg
Unescape View File

@ -15,9 +15,15 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh"
build_file: "grpc/tools/internal_ci/linux/grpc_xds.sh"
timeout_mins: 90
env_vars {
key: "BAZEL_SCRIPT"
value: "tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh"
}
action {
define_artifacts {
regex: "**/*sponge_log.*"
regex: "github/grpc/reports/**"
}
}

26
tools/internal_ci/linux/grpc_xds.sh
Unescape View File

@ -0,0 +1,26 @@
#!/usr/bin/env bash
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# change to grpc repo root
cd $(dirname $0)/../../..
source tools/internal_ci/helper_scripts/prepare_build_linux_rc
export DOCKERFILE_DIR=tools/dockerfile/test/bazel
export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT
export OUTPUT_DIR=reports
exec tools/run_tests/dockerize/build_and_run_docker.sh

8
tools/internal_ci/linux/grpc_xds_python.cfg
Unescape View File

@ -15,9 +15,15 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh"
build_file: "grpc/tools/internal_ci/linux/grpc_xds.sh"
timeout_mins: 90
env_vars {
key: "BAZEL_SCRIPT"
value: "tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh"
}
action {
define_artifacts {
regex: "**/*sponge_log.*"
regex: "github/grpc/reports/**"
}
}

6
tools/run_tests/python_utils/report_utils.py
Unescape View File

@ -31,9 +31,9 @@ def _filter_msg(msg, output_format):
if output_format in ['XML', 'HTML']:
# keep whitespaces but remove formfeed and vertical tab characters
# that make XML report unparsable.
filtered_msg = filter(
lambda x: x in string.printable and x != '\f' and x != '\v',
msg.decode('UTF-8', 'ignore'))
filtered_msg = ''.join(
filter(lambda x: x in string.printable and x != '\f' and x != '\v',
msg.decode('UTF-8', 'ignore')))
if output_format == 'HTML':
filtered_msg = filtered_msg.replace('"', '"')
return filtered_msg

160
tools/run_tests/run_xds_tests.py
Unescape View File

@ -29,6 +29,9 @@ import time
from oauth2client.client import GoogleCredentials
import python_utils.jobset as jobset
import python_utils.report_utils as report_utils
from src.proto.grpc.testing import messages_pb2
from src.proto.grpc.testing import test_pb2_grpc
@ -38,6 +41,26 @@ formatter = logging.Formatter(fmt='%(asctime)s: %(levelname)-8s %(message)s')
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
_TEST_CASES = [
'backends_restart',
'change_backend_service',
'new_instance_group_receives_traffic',
'ping_pong',
'remove_instance_group',
'round_robin',
'secondary_locality_gets_no_requests_on_partial_primary_failure',
'secondary_locality_gets_requests_on_primary_failure',
]
def parse_test_cases(arg):
if arg == 'all':
return _TEST_CASES
test_cases = arg.split(',')
if all([test_case in _TEST_CASES for test_case in test_cases]):
return test_cases
raise Exception('Failed to parse test cases %s' % arg)
def parse_port_range(port_arg):
try:
@ -58,17 +81,9 @@ argp.add_argument(
argp.add_argument(
'--test_case',
default='ping_pong',
choices=[
'all',
'backends_restart',
'change_backend_service',
'new_instance_group_receives_traffic',
'ping_pong',
'remove_instance_group',
'round_robin',
'secondary_locality_gets_no_requests_on_partial_primary_failure',
'secondary_locality_gets_requests_on_primary_failure',
])
type=parse_test_cases,
help='Comma-separated list of test cases to run, or \'all\' to run every '
'test. Available tests: %s' % ' '.join(_TEST_CASES))
argp.add_argument(
'--client_cmd',
default=None,
@ -183,6 +198,10 @@ _BASE_URL_MAP_NAME = 'test-map'
_BASE_SERVICE_HOST = 'grpc-test'
_BASE_TARGET_PROXY_NAME = 'test-target-proxy'
_BASE_FORWARDING_RULE_NAME = 'test-forwarding-rule'
_TEST_LOG_BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../../reports')
_SPONGE_LOG_NAME = 'sponge_log.log'
_SPONGE_XML_NAME = 'sponge_log.xml'
def get_client_stats(num_rpcs, timeout_sec):
@ -905,8 +924,6 @@ if args.compute_discovery_document:
else:
compute = googleapiclient.discovery.build('compute', 'v1')
client_process = None
try:
gcp = GcpState(compute, args.project_id)
health_check_name = _BASE_HEALTH_CHECK_NAME + args.gcp_suffix
@ -1034,56 +1051,75 @@ try:
server_uri = service_host_name
else:
server_uri = service_host_name + ':' + str(gcp.service_port)
cmd = args.client_cmd.format(server_uri=server_uri,
stats_port=args.stats_port,
qps=args.qps)
client_process = start_xds_client(cmd)
if args.test_case == 'all':
test_backends_restart(gcp, backend_service, instance_group)
test_change_backend_service(gcp, backend_service, instance_group,
alternate_backend_service,
same_zone_instance_group)
test_new_instance_group_receives_traffic(gcp, backend_service,
instance_group,
same_zone_instance_group)
test_ping_pong(gcp, backend_service, instance_group)
test_remove_instance_group(gcp, backend_service, instance_group,
same_zone_instance_group)
test_round_robin(gcp, backend_service, instance_group)
test_secondary_locality_gets_no_requests_on_partial_primary_failure(
gcp, backend_service, instance_group, secondary_zone_instance_group)
test_secondary_locality_gets_requests_on_primary_failure(
gcp, backend_service, instance_group, secondary_zone_instance_group)
elif args.test_case == 'backends_restart':
test_backends_restart(gcp, backend_service, instance_group)
elif args.test_case == 'change_backend_service':
test_change_backend_service(gcp, backend_service, instance_group,
alternate_backend_service,
same_zone_instance_group)
elif args.test_case == 'new_instance_group_receives_traffic':
test_new_instance_group_receives_traffic(gcp, backend_service,
instance_group,
same_zone_instance_group)
elif args.test_case == 'ping_pong':
test_ping_pong(gcp, backend_service, instance_group)
elif args.test_case == 'remove_instance_group':
test_remove_instance_group(gcp, backend_service, instance_group,
same_zone_instance_group)
elif args.test_case == 'round_robin':
test_round_robin(gcp, backend_service, instance_group)
elif args.test_case == 'secondary_locality_gets_no_requests_on_partial_primary_failure':
test_secondary_locality_gets_no_requests_on_partial_primary_failure(
gcp, backend_service, instance_group, secondary_zone_instance_group)
elif args.test_case == 'secondary_locality_gets_requests_on_primary_failure':
test_secondary_locality_gets_requests_on_primary_failure(
gcp, backend_service, instance_group, secondary_zone_instance_group)
else:
logger.error('Unknown test case: %s', args.test_case)
sys.exit(1)
with tempfile.NamedTemporaryFile(delete=False) as bootstrap_file:
bootstrap_file.write(
_BOOTSTRAP_TEMPLATE.format(
node_id=socket.gethostname()).encode('utf-8'))
bootstrap_path = bootstrap_file.name
client_env = dict(os.environ, GRPC_XDS_BOOTSTRAP=bootstrap_path)
client_cmd = shlex.split(
args.client_cmd.format(server_uri=server_uri,
stats_port=args.stats_port,
qps=args.qps))
test_results = {}
for test_case in args.test_case:
result = jobset.JobResult()
log_dir = os.path.join(_TEST_LOG_BASE_DIR, test_case)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
test_log_file = open(os.path.join(log_dir, _SPONGE_LOG_NAME), 'w+')
try:
client_process = subprocess.Popen(client_cmd,
env=client_env,
stderr=subprocess.STDOUT,
stdout=test_log_file)
if test_case == 'backends_restart':
test_backends_restart(gcp, backend_service, instance_group)
elif test_case == 'change_backend_service':
test_change_backend_service(gcp, backend_service,
instance_group,
alternate_backend_service,
same_zone_instance_group)
elif test_case == 'new_instance_group_receives_traffic':
test_new_instance_group_receives_traffic(
gcp, backend_service, instance_group,
same_zone_instance_group)
elif test_case == 'ping_pong':
test_ping_pong(gcp, backend_service, instance_group)
elif test_case == 'remove_instance_group':
test_remove_instance_group(gcp, backend_service, instance_group,
same_zone_instance_group)
elif test_case == 'round_robin':
test_round_robin(gcp, backend_service, instance_group)
elif test_case == 'secondary_locality_gets_no_requests_on_partial_primary_failure':
test_secondary_locality_gets_no_requests_on_partial_primary_failure(
gcp, backend_service, instance_group,
secondary_zone_instance_group)
elif test_case == 'secondary_locality_gets_requests_on_primary_failure':
test_secondary_locality_gets_requests_on_primary_failure(
gcp, backend_service, instance_group,
secondary_zone_instance_group)
else:
logger.error('Unknown test case: %s', test_case)
sys.exit(1)
result.state = 'PASSED'
result.returncode = 0
except Exception as e:
result.state = 'FAILED'
result.message = str(e).encode('UTF-8')
finally:
if client_process:
client_process.terminate()
test_results[test_case] = [result]
if not os.path.exists(_TEST_LOG_BASE_DIR):
os.makedirs(_TEST_LOG_BASE_DIR)
report_utils.render_junit_xml_report(test_results,
os.path.join(_TEST_LOG_BASE_DIR,
_SPONGE_XML_NAME),
suite_name='xds_tests',
multi_target=True)
finally:
if client_process:
client_process.terminate()
if not args.keep_gcp_resources:
logger.info('Cleaning up GCP resources. This may take some time.')
clean_up(gcp)

Write Preview
Loading…
Cancel
Save