From f62140d56a980fde27ceab5ee059cf2450fa5c88 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 17 Oct 2016 12:16:14 -0700 Subject: [PATCH] improve pull request test filtering --- tools/run_tests/filter_pull_request_tests.py | 110 ++++++++++--------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/tools/run_tests/filter_pull_request_tests.py b/tools/run_tests/filter_pull_request_tests.py index 55dab42f8a7..128370e1914 100644 --- a/tools/run_tests/filter_pull_request_tests.py +++ b/tools/run_tests/filter_pull_request_tests.py @@ -57,44 +57,68 @@ class TestSuite: self.triggers.append(trigger) # Create test suites -_core_test_suite = TestSuite(['_c_']) -_cpp_test_suite = TestSuite(['_c++_']) -_csharp_test_suite = TestSuite(['_csharp_']) -_node_test_suite = TestSuite(['_node_']) -_objc_test_suite = TestSuite(['_objc_']) -_php_test_suite = TestSuite(['_php_', '_php7_']) -_python_test_suite = TestSuite(['_python_']) -_ruby_test_suite = TestSuite(['_ruby']) -_all_test_suites = [_core_test_suite, _cpp_test_suite, _csharp_test_suite, - _node_test_suite, _objc_test_suite, _php_test_suite, - _python_test_suite, _ruby_test_suite] +_CORE_TEST_SUITE = TestSuite(['_c_']) +_CPP_TEST_SUITE = TestSuite(['_c++_']) +_CSHARP_TEST_SUITE = TestSuite(['_csharp_']) +_NODE_TEST_SUITE = TestSuite(['_node_']) +_OBJC_TEST_SUITE = TestSuite(['_objc_']) +_PHP_TEST_SUITE = TestSuite(['_php_', '_php7_']) +_PYTHON_TEST_SUITE = TestSuite(['_python_']) +_RUBY_TEST_SUITE = TestSuite(['_ruby']) +_ALL_TEST_SUITES = [_CORE_TEST_SUITE, _CPP_TEST_SUITE, _CSHARP_TEST_SUITE, + _NODE_TEST_SUITE, _OBJC_TEST_SUITE, _PHP_TEST_SUITE, + _PYTHON_TEST_SUITE, _RUBY_TEST_SUITE] # Dictionary of whitelistable files where the key is a regex matching changed files # and the value is a list of tests that should be run. An empty list means that # the changed files should not trigger any tests. Any changed file that does not # match any of these regexes will trigger all tests _WHITELIST_DICT = { - '^templates/.*': [], - '^doc/.*': [], - '^examples/.*': [], - '^summerofcode/.*': [], - '.*README.md$': [], - '.*LICENSE$': [], - '^src/cpp.*': [_cpp_test_suite], - '^src/csharp.*': [_csharp_test_suite], - '^src/node.*': [_node_test_suite], - '^src/objective-c.*': [_objc_test_suite], - '^src/php.*': [_php_test_suite], - '^src/python.*': [_python_test_suite], - '^src/ruby.*': [_ruby_test_suite], - '^test/core.*': [_core_test_suite], - '^test/cpp.*': [_cpp_test_suite], - '^test/distrib/cpp.*': [_cpp_test_suite], - '^test/distrib/csharp.*': [_csharp_test_suite], - '^test/distrib/node.*': [_node_test_suite], - '^test/distrib/php.*': [_php_test_suite], - '^test/distrib/python.*': [_python_test_suite], - '^test/distrib/ruby.*': [_ruby_test_suite] + #'^templates/.*': [_sanity_test_suite], + # todo(mattkwong): add sanity test suite + '^doc/': [], + '^examples/': [], + '^summerofcode/': [], + 'README\.md$': [], + 'CONTRIBUTING\.md$': [], + 'LICENSE$': [], + 'INSTALL\.md$': [], + 'MANIFEST\.md$': [], + 'PATENTS$': [], + 'binding\.grp$': [_NODE_TEST_SUITE], + 'gRPC\-Core\.podspec$': [_OBJC_TEST_SUITE], + 'gRPC\-ProtoRPC\.podspec$': [_OBJC_TEST_SUITE], + 'gRPC\-RxLibrary\.podspec$': [_OBJC_TEST_SUITE], + 'gRPC\.podspec$': [_OBJC_TEST_SUITE], + 'composer\.json$': [_PHP_TEST_SUITE], + 'config\.m4$': [_PHP_TEST_SUITE], + 'package\.json$': [_PHP_TEST_SUITE], + 'package\.xml$': [_PHP_TEST_SUITE], + 'PYTHON\-MANIFEST\.in$': [_PYTHON_TEST_SUITE], + 'requirements\.txt$': [_PYTHON_TEST_SUITE], + 'setup\.cfg$': [_PYTHON_TEST_SUITE], + 'setup\.py$': [_PYTHON_TEST_SUITE], + 'grpc\.gemspec$': [_RUBY_TEST_SUITE], + 'Gemfile$': [_RUBY_TEST_SUITE], + # 'grpc.def$': [_WINDOWS_TEST_SUITE], + '^src/cpp/': [_CPP_TEST_SUITE], + '^src/csharp/': [_CSHARP_TEST_SUITE], + '^src/node/': [_NODE_TEST_SUITE], + '^src/objective\-c/': [_OBJC_TEST_SUITE], + '^src/php/': [_PHP_TEST_SUITE], + '^src/python/': [_PYTHON_TEST_SUITE], + '^src/ruby/': [_RUBY_TEST_SUITE], + '^test/core/': [_CORE_TEST_SUITE], + '^test/cpp/': [_CPP_TEST_SUITE], + '^test/distrib/cpp/': [_CPP_TEST_SUITE], + '^test/distrib/csharp/': [_CSHARP_TEST_SUITE], + '^test/distrib/node/': [_NODE_TEST_SUITE], + '^test/distrib/php/': [_PHP_TEST_SUITE], + '^test/distrib/python/': [_PYTHON_TEST_SUITE], + '^test/distrib/ruby/': [_RUBY_TEST_SUITE], + '^include/grpc\+\+/': [_CPP_TEST_SUITE] + #'^vsprojects/': [_WINDOWS_TEST_SUITE] + # todo(mattkwong): add windows test suite } # Add all triggers to their respective test suites for trigger, test_suites in _WHITELIST_DICT.iteritems(): @@ -108,7 +132,7 @@ def _get_changed_files(base_branch): """ # git fetch might need to be called on Jenkins slave # todo(mattkwong): remove or uncomment below after seeing if Jenkins needs this - # call(['git', 'fetch']) + call(['git', 'fetch']) # Get file changes between branch and merge-base of specified branch # Not combined to be Windows friendly @@ -137,19 +161,7 @@ def _remove_irrelevant_tests(tests, tag): :return: list of relevant tests """ # todo(mattkwong): find a more reliable way to filter tests - don't use shortname - return [test for test in tests if tag not in test.shortname or - any(san_tag in test.shortname for san_tag in ['_asan', '_tsan', '_msan'])] - - -def _remove_sanitizer_tests(tests): - """ - Filters out sanitizer tests - :param tests: list of all tests generated by run_tests_matrix.py - :return: list of relevant tests - """ - # todo(mattkwong): find a more reliable way to filter tests - don't use shortname - return [test for test in tests if - all(san_tag not in test.shortname for san_tag in ['_asan', '_tsan', '_msan'])] + return [test for test in tests if tag not in test.shortname] def filter_tests(tests, base_branch): @@ -171,14 +183,10 @@ def filter_tests(tests, base_branch): if not re.match(all_triggers, changed_file): return(tests) # Filter out tests by language - for test_suite in _all_test_suites: + for test_suite in _ALL_TEST_SUITES: if _can_skip_tests(changed_files, test_suite.triggers): for tag in test_suite.tags: print(" Filtering %s tests" % tag) tests = _remove_irrelevant_tests(tests, tag) - # Sanitizer tests skipped if core and c++ are skipped - if _can_skip_tests(changed_files, _cpp_test_suite.triggers + _core_test_suite.triggers): - print(" Filtering Sanitizer tests") - tests = _remove_sanitizer_tests(tests) return tests