grpc/tools/interop_matrix/run_interop_matrix_tests.py

#!/usr/bin/env python2.7
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Run tests using docker images in Google Container Registry per matrix."""

from __future__ import print_function

import argparse
import atexit
import json
import multiprocessing
import os
import re
import subprocess
import sys
import uuid

# Langauage Runtime Matrix
import client_matrix

python_util_dir = os.path.abspath(
    os.path.join(os.path.dirname(__file__), '../run_tests/python_utils'))
sys.path.append(python_util_dir)
import dockerjob
import jobset
import report_utils
import upload_test_results

_LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys()
# All gRPC release tags, flattened, deduped and sorted.
_RELEASES = sorted(
    list(
        set(
            client_matrix.get_release_tag_name(info)
            for lang in client_matrix.LANG_RELEASE_MATRIX.values()
            for info in lang)))
_TEST_TIMEOUT = 60

argp = argparse.ArgumentParser(description='Run interop tests.')
argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
argp.add_argument(
    '--gcr_path',
    default='gcr.io/grpc-testing',
    help='Path of docker images in Google Container Registry')
argp.add_argument(
    '--release',
    default='all',
    choices=['all', 'master'] + _RELEASES,
    help='Release tags to test.  When testing all '
    'releases defined in client_matrix.py, use "all".')
argp.add_argument(
    '-l',
    '--language',
    choices=['all'] + sorted(_LANGUAGES),
    nargs='+',
    default=['all'],
    help='Languages to test')
argp.add_argument(
    '--keep',
    action='store_true',
    help='keep the created local images after finishing the tests.')
argp.add_argument(
    '--report_file', default='report.xml', help='The result file to create.')
argp.add_argument(
    '--allow_flakes',
    default=False,
    action='store_const',
    const=True,
    help=('Allow flaky tests to show as passing (re-runs failed '
          'tests up to five times)'))
argp.add_argument(
    '--bq_result_table',
    default='',
    type=str,
    nargs='?',
    help='Upload test results to a specified BQ table.')
argp.add_argument(
    '--server_host',
    default='74.125.206.210',
    type=str,
    nargs='?',
    help='The gateway to backend services.')

args = argp.parse_args()

print(str(args))


def find_all_images_for_lang(lang):
    """Find docker images for a language across releases and runtimes.

  Returns dictionary of list of (<tag>, <image-full-path>) keyed by runtime.
  """
    # Find all defined releases.
    if args.release == 'all':
        releases = ['master'] + client_matrix.get_release_tags(lang)
    else:
        # Look for a particular release.
        if args.release not in ['master'
                               ] + client_matrix.get_release_tags(lang):
            jobset.message(
                'SKIPPED',
                '%s for %s is not defined' % (args.release, lang),
                do_newline=True)
            return {}
        releases = [args.release]

    # TODO(jtattermusch): why do we need to query the existing images/tags?
    # From LANG_RUNTIME_MATRIX and LANG_RELEASE_MATRIX it should be obvious
    # which tags we want to test - and it should be an error if they are
    # missing.
    # Images tuples keyed by runtime.
    images = {}
    for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
        image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime)
        output = subprocess.check_output([
            'gcloud', 'beta', 'container', 'images', 'list-tags',
            '--format=json', image_path
        ])
        docker_image_list = json.loads(output)
        # All images should have a single tag or no tag.
        # TODO(adelez): Remove tagless images.
        tags = [i['tags'][0] for i in docker_image_list if i['tags']]
        jobset.message(
            'START',
            'Found images for %s: %s' % (image_path, tags),
            do_newline=True)
        skipped = len(docker_image_list) - len(tags)
        jobset.message(
            'SKIPPED',
            'Skipped images (no-tag/unknown-tag): %d' % skipped,
            do_newline=True)
        # Filter tags based on the releases.
        images[runtime] = [(tag, '%s:%s' % (image_path, tag))
                           for tag in tags
                           if tag in releases]
    return images


# caches test cases (list of JobSpec) loaded from file.  Keyed by lang and runtime.
def find_test_cases(lang, runtime, release, suite_name):
    """Returns the list of test cases from testcase files per lang/release."""
    testcase_dir = os.path.join(os.path.dirname(__file__), 'testcases')
    filename_prefix = lang
    if lang == 'csharp':
        filename_prefix = runtime
    # Check to see if we need to use a particular version of test cases.
    lang_version = '%s_%s' % (filename_prefix, release)
    if lang_version in client_matrix.TESTCASES_VERSION_MATRIX:
        testcases = os.path.join(
            testcase_dir, client_matrix.TESTCASES_VERSION_MATRIX[lang_version])
    else:
        testcases = os.path.join(testcase_dir, '%s__master' % filename_prefix)

    job_spec_list = []
    try:
        with open(testcases) as f:
            # Only line start with 'docker run' are test cases.
            for line in f.readlines():
                if line.startswith('docker run'):
                    m = re.search('--test_case=(.*)"', line)
                    shortname = m.group(1) if m else 'unknown_test'
                    m = re.search(
                        '--server_host_override=(.*).sandbox.googleapis.com',
                        line)
                    server = m.group(1) if m else 'unknown_server'

                    # If server_host arg is not None, replace the original
                    # server_host with the one provided or append to the end of
                    # the command if server_host does not appear originally.
                    if args.server_host:
                        if line.find('--server_host=') > -1:
                            line = re.sub('--server_host=[^ ]*',
                                          '--server_host=%s' % args.server_host,
                                          line)
                        else:
                            line = '%s --server_host=%s"' % (line[:-1],
                                                             args.server_host)
                        print(line)

                    spec = jobset.JobSpec(
                        cmdline=line,
                        shortname='%s:%s:%s:%s' % (suite_name, lang, server,
                                                   shortname),
                        timeout_seconds=_TEST_TIMEOUT,
                        shell=True,
                        flake_retries=5 if args.allow_flakes else 0)
                    job_spec_list.append(spec)
            jobset.message(
                'START',
                'Loaded %s tests from %s' % (len(job_spec_list), testcases),
                do_newline=True)
    except IOError as err:
        jobset.message('FAILED', err, do_newline=True)
    return job_spec_list


_xml_report_tree = report_utils.new_junit_xml_tree()


def run_tests_for_lang(lang, runtime, images):
    """Find and run all test cases for a language.

  images is a list of (<release-tag>, <image-full-path>) tuple.
  """
    total_num_failures = 0
    for image_tuple in images:
        release, image = image_tuple
        jobset.message('START', 'Testing %s' % image, do_newline=True)
        # Download the docker image before running each test case.
        subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
        suite_name = '%s__%s_%s' % (lang, runtime, release)
        job_spec_list = find_test_cases(lang, runtime, release, suite_name)

        if not job_spec_list:
            jobset.message(
                'FAILED', 'No test cases were found.', do_newline=True)
            return 1

        num_failures, resultset = jobset.run(
            job_spec_list,
            newline_on_success=True,
            add_env={'docker_image': image},
            maxjobs=args.jobs)
        if args.bq_result_table and resultset:
            upload_test_results.upload_interop_results_to_bq(
                resultset, args.bq_result_table, args)
        if num_failures:
            jobset.message('FAILED', 'Some tests failed', do_newline=True)
            total_num_failures += num_failures
        else:
            jobset.message('SUCCESS', 'All tests passed', do_newline=True)

        report_utils.append_junit_xml_results(_xml_report_tree, resultset,
                                              'grpc_interop_matrix', suite_name,
                                              str(uuid.uuid4()))

        if not args.keep:
            cleanup(image)

    return total_num_failures


def cleanup(image):
    jobset.message('START', 'Cleanup docker image %s' % image, do_newline=True)
    dockerjob.remove_image(image, skip_nonexistent=True)


languages = args.language if args.language != ['all'] else _LANGUAGES
total_num_failures = 0
for lang in languages:
    docker_images = find_all_images_for_lang(lang)
    for runtime in sorted(docker_images.keys()):
        total_num_failures += run_tests_for_lang(lang, runtime,
                                                 docker_images[runtime])

report_utils.create_xml_report_file(_xml_report_tree, args.report_file)

if total_num_failures:
    sys.exit(1)
sys.exit(0)