diff --git a/tools/internal_ci/linux/grpc_xds.cfg b/tools/internal_ci/linux/grpc_xds.cfg index 888be05cde5..107404a30e9 100644 --- a/tools/internal_ci/linux/grpc_xds.cfg +++ b/tools/internal_ci/linux/grpc_xds.cfg @@ -15,9 +15,15 @@ # Config file for the internal CI (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" +build_file: "grpc/tools/internal_ci/linux/grpc_xds.sh" timeout_mins: 90 env_vars { key: "BAZEL_SCRIPT" value: "tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh" } +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/linux/grpc_xds.sh b/tools/internal_ci/linux/grpc_xds.sh new file mode 100755 index 00000000000..6de6588024c --- /dev/null +++ b/tools/internal_ci/linux/grpc_xds.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +export DOCKERFILE_DIR=tools/dockerfile/test/bazel +export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT +export OUTPUT_DIR=reports +exec tools/run_tests/dockerize/build_and_run_docker.sh diff --git a/tools/internal_ci/linux/grpc_xds_python.cfg b/tools/internal_ci/linux/grpc_xds_python.cfg index e635b334dbd..922b1aa88a1 100644 --- a/tools/internal_ci/linux/grpc_xds_python.cfg +++ b/tools/internal_ci/linux/grpc_xds_python.cfg @@ -15,9 +15,15 @@ # Config file for the internal CI (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" +build_file: "grpc/tools/internal_ci/linux/grpc_xds.sh" timeout_mins: 90 env_vars { key: "BAZEL_SCRIPT" value: "tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh" } +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py index 43aaf50af04..666edbf63d6 100644 --- a/tools/run_tests/python_utils/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py @@ -31,9 +31,9 @@ def _filter_msg(msg, output_format): if output_format in ['XML', 'HTML']: # keep whitespaces but remove formfeed and vertical tab characters # that make XML report unparsable. - filtered_msg = filter( - lambda x: x in string.printable and x != '\f' and x != '\v', - msg.decode('UTF-8', 'ignore')) + filtered_msg = ''.join( + filter(lambda x: x in string.printable and x != '\f' and x != '\v', + msg.decode('UTF-8', 'ignore'))) if output_format == 'HTML': filtered_msg = filtered_msg.replace('"', '"') return filtered_msg diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 21e09e550d2..f518d12e06f 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -29,6 +29,9 @@ import time from oauth2client.client import GoogleCredentials +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils + from src.proto.grpc.testing import messages_pb2 from src.proto.grpc.testing import test_pb2_grpc @@ -38,6 +41,26 @@ formatter = logging.Formatter(fmt='%(asctime)s: %(levelname)-8s %(message)s') console_handler.setFormatter(formatter) logger.addHandler(console_handler) +_TEST_CASES = [ + 'backends_restart', + 'change_backend_service', + 'new_instance_group_receives_traffic', + 'ping_pong', + 'remove_instance_group', + 'round_robin', + 'secondary_locality_gets_no_requests_on_partial_primary_failure', + 'secondary_locality_gets_requests_on_primary_failure', +] + + +def parse_test_cases(arg): + if arg == 'all': + return _TEST_CASES + test_cases = arg.split(',') + if all([test_case in _TEST_CASES for test_case in test_cases]): + return test_cases + raise Exception('Failed to parse test cases %s' % arg) + def parse_port_range(port_arg): try: @@ -58,17 +81,9 @@ argp.add_argument( argp.add_argument( '--test_case', default='ping_pong', - choices=[ - 'all', - 'backends_restart', - 'change_backend_service', - 'new_instance_group_receives_traffic', - 'ping_pong', - 'remove_instance_group', - 'round_robin', - 'secondary_locality_gets_no_requests_on_partial_primary_failure', - 'secondary_locality_gets_requests_on_primary_failure', - ]) + type=parse_test_cases, + help='Comma-separated list of test cases to run, or \'all\' to run every ' + 'test. Available tests: %s' % ' '.join(_TEST_CASES)) argp.add_argument( '--client_cmd', default=None, @@ -183,6 +198,10 @@ _BASE_URL_MAP_NAME = 'test-map' _BASE_SERVICE_HOST = 'grpc-test' _BASE_TARGET_PROXY_NAME = 'test-target-proxy' _BASE_FORWARDING_RULE_NAME = 'test-forwarding-rule' +_TEST_LOG_BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), + '../../reports') +_SPONGE_LOG_NAME = 'sponge_log.log' +_SPONGE_XML_NAME = 'sponge_log.xml' def get_client_stats(num_rpcs, timeout_sec): @@ -905,8 +924,6 @@ if args.compute_discovery_document: else: compute = googleapiclient.discovery.build('compute', 'v1') -client_process = None - try: gcp = GcpState(compute, args.project_id) health_check_name = _BASE_HEALTH_CHECK_NAME + args.gcp_suffix @@ -1034,56 +1051,75 @@ try: server_uri = service_host_name else: server_uri = service_host_name + ':' + str(gcp.service_port) - cmd = args.client_cmd.format(server_uri=server_uri, - stats_port=args.stats_port, - qps=args.qps) - client_process = start_xds_client(cmd) - - if args.test_case == 'all': - test_backends_restart(gcp, backend_service, instance_group) - test_change_backend_service(gcp, backend_service, instance_group, - alternate_backend_service, - same_zone_instance_group) - test_new_instance_group_receives_traffic(gcp, backend_service, - instance_group, - same_zone_instance_group) - test_ping_pong(gcp, backend_service, instance_group) - test_remove_instance_group(gcp, backend_service, instance_group, - same_zone_instance_group) - test_round_robin(gcp, backend_service, instance_group) - test_secondary_locality_gets_no_requests_on_partial_primary_failure( - gcp, backend_service, instance_group, secondary_zone_instance_group) - test_secondary_locality_gets_requests_on_primary_failure( - gcp, backend_service, instance_group, secondary_zone_instance_group) - elif args.test_case == 'backends_restart': - test_backends_restart(gcp, backend_service, instance_group) - elif args.test_case == 'change_backend_service': - test_change_backend_service(gcp, backend_service, instance_group, - alternate_backend_service, - same_zone_instance_group) - elif args.test_case == 'new_instance_group_receives_traffic': - test_new_instance_group_receives_traffic(gcp, backend_service, - instance_group, - same_zone_instance_group) - elif args.test_case == 'ping_pong': - test_ping_pong(gcp, backend_service, instance_group) - elif args.test_case == 'remove_instance_group': - test_remove_instance_group(gcp, backend_service, instance_group, - same_zone_instance_group) - elif args.test_case == 'round_robin': - test_round_robin(gcp, backend_service, instance_group) - elif args.test_case == 'secondary_locality_gets_no_requests_on_partial_primary_failure': - test_secondary_locality_gets_no_requests_on_partial_primary_failure( - gcp, backend_service, instance_group, secondary_zone_instance_group) - elif args.test_case == 'secondary_locality_gets_requests_on_primary_failure': - test_secondary_locality_gets_requests_on_primary_failure( - gcp, backend_service, instance_group, secondary_zone_instance_group) - else: - logger.error('Unknown test case: %s', args.test_case) - sys.exit(1) + with tempfile.NamedTemporaryFile(delete=False) as bootstrap_file: + bootstrap_file.write( + _BOOTSTRAP_TEMPLATE.format( + node_id=socket.gethostname()).encode('utf-8')) + bootstrap_path = bootstrap_file.name + client_env = dict(os.environ, GRPC_XDS_BOOTSTRAP=bootstrap_path) + client_cmd = shlex.split( + args.client_cmd.format(server_uri=server_uri, + stats_port=args.stats_port, + qps=args.qps)) + + test_results = {} + for test_case in args.test_case: + result = jobset.JobResult() + log_dir = os.path.join(_TEST_LOG_BASE_DIR, test_case) + if not os.path.exists(log_dir): + os.makedirs(log_dir) + test_log_file = open(os.path.join(log_dir, _SPONGE_LOG_NAME), 'w+') + try: + client_process = subprocess.Popen(client_cmd, + env=client_env, + stderr=subprocess.STDOUT, + stdout=test_log_file) + if test_case == 'backends_restart': + test_backends_restart(gcp, backend_service, instance_group) + elif test_case == 'change_backend_service': + test_change_backend_service(gcp, backend_service, + instance_group, + alternate_backend_service, + same_zone_instance_group) + elif test_case == 'new_instance_group_receives_traffic': + test_new_instance_group_receives_traffic( + gcp, backend_service, instance_group, + same_zone_instance_group) + elif test_case == 'ping_pong': + test_ping_pong(gcp, backend_service, instance_group) + elif test_case == 'remove_instance_group': + test_remove_instance_group(gcp, backend_service, instance_group, + same_zone_instance_group) + elif test_case == 'round_robin': + test_round_robin(gcp, backend_service, instance_group) + elif test_case == 'secondary_locality_gets_no_requests_on_partial_primary_failure': + test_secondary_locality_gets_no_requests_on_partial_primary_failure( + gcp, backend_service, instance_group, + secondary_zone_instance_group) + elif test_case == 'secondary_locality_gets_requests_on_primary_failure': + test_secondary_locality_gets_requests_on_primary_failure( + gcp, backend_service, instance_group, + secondary_zone_instance_group) + else: + logger.error('Unknown test case: %s', test_case) + sys.exit(1) + result.state = 'PASSED' + result.returncode = 0 + except Exception as e: + result.state = 'FAILED' + result.message = str(e).encode('UTF-8') + finally: + if client_process: + client_process.terminate() + test_results[test_case] = [result] + if not os.path.exists(_TEST_LOG_BASE_DIR): + os.makedirs(_TEST_LOG_BASE_DIR) + report_utils.render_junit_xml_report(test_results, + os.path.join(_TEST_LOG_BASE_DIR, + _SPONGE_XML_NAME), + suite_name='xds_tests', + multi_target=True) finally: - if client_process: - client_process.terminate() if not args.keep_gcp_resources: logger.info('Cleaning up GCP resources. This may take some time.') clean_up(gcp)