diff --git a/tools/run_tests/filter_pull_request_tests.py b/tools/run_tests/filter_pull_request_tests.py new file mode 100644 index 00000000000..6cc06d9c409 --- /dev/null +++ b/tools/run_tests/filter_pull_request_tests.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python2.7 +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Filter out tests based on file differences compared to grpc:master""" + +from subprocess import call, check_output + +# triggers to skip c++ tests +run_cpp_starts_with_triggers = ('src/core', + 'src/cpp', + 'test/core', + 'test/cpp') + + +def _get_changed_files(): + """ + Get list of changed files between current branch and gRPC master branch + """ + # git fetch might need to be called on Jenkins slave + # todo(mattkwong): remove or uncomment below after seeing if Jenkins needs this + # call(['git', 'fetch']) + # this also collects files that are changed in the repo but not updated in the branch + return check_output(["git", "diff", "--name-only", "..origin/master"]).split() + + +def _can_skip_tests(file_names, starts_with_triggers=(), ends_with_triggers=()): + """ + Determines if tests are skippable based on if all file names do not match + any begin or end triggers + :param file_names: list of changed files generated by _get_changed_files() + :param starts_with_triggers: tuple of strings to match with beginning of file names + :param ends_with_triggers: tuple of strings to match with end of file names + :return: safe to skip tests + """ + for file_name in file_names: + if starts_with_triggers and file_name.startswith(starts_with_triggers) or \ + ends_with_triggers and file_name.endswith(ends_with_triggers): + return False + return True + + +def _remove_irrelevant_tests(tests, tag): + """ + Filters out tests by config or language + :param tests: list of all tests generated by run_tests_matrix.py + :param tag: string representing language or config to filter - "_(language)_" or "_(config)" + :return: list of relevant tests + """ + return [test for test in tests if not tag in test.shortname] + + +def filter_tests(tests): + """ + Filters out tests that are safe to ignore + :param tests: list of all tests generated by run_tests_matrix.py + :return: list of relevant tests + """ + print("Finding file differences between grpc:master repo and pull request...") + changed_files = _get_changed_files() + for changed_file in changed_files: + print(changed_file) + # C++, tsan, msan, and asan have the same filter + if _can_skip_tests(changed_files, starts_with_triggers=run_cpp_starts_with_triggers): + tests = _remove_irrelevant_tests(tests, '_c++_') # filter out c++ tests + tests = _remove_irrelevant_tests(tests, '_tsan') # filter out tsan tests + tests = _remove_irrelevant_tests(tests, '_msan') # filter out msan tests + tests = _remove_irrelevant_tests(tests, '_asan') # filter out asan tests + return tests diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 60c21a1e218..235bd162be0 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -36,6 +36,7 @@ import multiprocessing import os import report_utils import sys +from filter_pull_request_tests import filter_tests _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) @@ -231,6 +232,11 @@ argp.add_argument('--dry_run', action='store_const', const=True, help='Only print what would be run.') +argp.add_argument('--filter_pr_tests', + default=False, + action='store_const', + const=True, + help='Filters out tests irrelavant to pull request changes.') args = argp.parse_args() extra_args = [] @@ -264,6 +270,18 @@ for job in jobs: print ' %s' % job.shortname print +if args.filter_pr_tests: + print 'IMPORTANT: Test filtering is not active; this is only for testing.' + relevant_jobs = filter_tests(jobs) + print + if len(relevant_jobs) == len(jobs): + print 'No tests were filtered.' + else: + print 'These tests were filtered:' + for job in list(set(jobs) - set(relevant_jobs)): + print ' %s' % job.shortname + print + if args.dry_run: print '--dry_run was used, exiting' sys.exit(1)