@ -36,18 +36,16 @@ from subprocess import call, check_output
class TestSuite :
"""
Contains tag to identify job as belonging to this test suite and
Contains label to identify job as belonging to this test suite and
triggers to identify if changed files are relevant
"""
def __init__ ( self , tag s) :
def __init__ ( self , label s) :
"""
Build TestSuite to group tests by their tags
: param tag : string used to identify if a job belongs to this TestSuite
todo ( mattkwong ) : Change the use of tag because do not want to depend on
job . shortname to identify what suite a test belongs to
Build TestSuite to group tests based on labeling
: param label : strings that should match a jobs ' s platform, config, language, or test group
"""
self . triggers = [ ]
self . tags = tag s
self . labels = label s
def add_trigger ( self , trigger ) :
"""
@ -56,46 +54,75 @@ class TestSuite:
"""
self . triggers . append ( trigger )
# Create test suites
_core_test_suite = TestSuite ( [ ' _c_ ' ] )
_cpp_test_suite = TestSuite ( [ ' _c++_ ' ] )
_csharp_test_suite = TestSuite ( [ ' _csharp_ ' ] )
_node_test_suite = TestSuite ( [ ' _node_ ' ] )
_objc_test_suite = TestSuite ( [ ' _objc_ ' ] )
_php_test_suite = TestSuite ( [ ' _php_ ' , ' _php7_ ' ] )
_python_test_suite = TestSuite ( [ ' _python_ ' ] )
_ruby_test_suite = TestSuite ( [ ' _ruby ' ] )
_all_test_suites = [ _core_test_suite , _cpp_test_suite , _csharp_test_suite ,
_node_test_suite , _objc_test_suite , _php_test_suite ,
_python_test_suite , _ruby_test_suite ]
_SANITY_TEST_SUITE = TestSuite ( [ ' sanity ' ] )
_CORE_TEST_SUITE = TestSuite ( [ ' c ' ] )
_CPP_TEST_SUITE = TestSuite ( [ ' c++ ' ] )
_CSHARP_TEST_SUITE = TestSuite ( [ ' csharp ' ] )
_NODE_TEST_SUITE = TestSuite ( [ ' node ' ] )
_OBJC_TEST_SUITE = TestSuite ( [ ' objc ' ] )
_PHP_TEST_SUITE = TestSuite ( [ ' php ' , ' php7 ' ] )
_PYTHON_TEST_SUITE = TestSuite ( [ ' python ' ] )
_RUBY_TEST_SUITE = TestSuite ( [ ' ruby ' ] )
_LINUX_TEST_SUITE = TestSuite ( [ ' linux ' ] )
_WINDOWS_TEST_SUITE = TestSuite ( [ ' windows ' ] )
_MACOS_TEST_SUITE = TestSuite ( [ ' macos ' ] )
_ALL_TEST_SUITES = [ _SANITY_TEST_SUITE , _CORE_TEST_SUITE , _CPP_TEST_SUITE ,
_CSHARP_TEST_SUITE , _NODE_TEST_SUITE , _OBJC_TEST_SUITE ,
_PHP_TEST_SUITE , _PYTHON_TEST_SUITE , _RUBY_TEST_SUITE ,
_LINUX_TEST_SUITE , _WINDOWS_TEST_SUITE , _MACOS_TEST_SUITE ]
# Dictionary of whitelistable files where the key is a regex matching changed files
# and the value is a list of tests that should be run. An empty list means that
# the changed files should not trigger any tests. Any changed file that does not
# match any of these regexes will trigger all tests
_WHITELIST_DICT = {
' ^templates/.* ' : [ ] ,
' ^doc/.* ' : [ ] ,
' ^examples/.* ' : [ ] ,
' ^summerofcode/.* ' : [ ] ,
' .*README.md$ ' : [ ] ,
' .*LICENSE$ ' : [ ] ,
' ^src/cpp.* ' : [ _cpp_test_suite ] ,
' ^src/csharp.* ' : [ _csharp_test_suite ] ,
' ^src/node.* ' : [ _node_test_suite ] ,
' ^src/objective-c.* ' : [ _objc_test_suite ] ,
' ^src/php.* ' : [ _php_test_suite ] ,
' ^src/python.* ' : [ _python_test_suite ] ,
' ^src/ruby.* ' : [ _ruby_test_suite ] ,
' ^test/core.* ' : [ _core_test_suite ] ,
' ^test/cpp.* ' : [ _cpp_test_suite ] ,
' ^test/distrib/cpp.* ' : [ _cpp_test_suite ] ,
' ^test/distrib/csharp.* ' : [ _csharp_test_suite ] ,
' ^test/distrib/node.* ' : [ _node_test_suite ] ,
' ^test/distrib/php.* ' : [ _php_test_suite ] ,
' ^test/distrib/python.* ' : [ _python_test_suite ] ,
' ^test/distrib/ruby.* ' : [ _ruby_test_suite ]
' ^doc/ ' : [ ] ,
' ^examples/ ' : [ ] ,
' ^include/grpc \ + \ +/ ' : [ _CPP_TEST_SUITE ] ,
' ^summerofcode/ ' : [ ] ,
' ^src/cpp/ ' : [ _CPP_TEST_SUITE ] ,
' ^src/csharp/ ' : [ _CSHARP_TEST_SUITE ] ,
' ^src/node/ ' : [ _NODE_TEST_SUITE ] ,
' ^src/objective \ -c/ ' : [ _OBJC_TEST_SUITE ] ,
' ^src/php/ ' : [ _PHP_TEST_SUITE ] ,
' ^src/python/ ' : [ _PYTHON_TEST_SUITE ] ,
' ^src/ruby/ ' : [ _RUBY_TEST_SUITE ] ,
' ^templates/ ' : [ _SANITY_TEST_SUITE ] ,
' ^test/core/ ' : [ _CORE_TEST_SUITE ] ,
' ^test/cpp/ ' : [ _CPP_TEST_SUITE ] ,
' ^test/distrib/cpp/ ' : [ _CPP_TEST_SUITE ] ,
' ^test/distrib/csharp/ ' : [ _CSHARP_TEST_SUITE ] ,
' ^test/distrib/node/ ' : [ _NODE_TEST_SUITE ] ,
' ^test/distrib/php/ ' : [ _PHP_TEST_SUITE ] ,
' ^test/distrib/python/ ' : [ _PYTHON_TEST_SUITE ] ,
' ^test/distrib/ruby/ ' : [ _RUBY_TEST_SUITE ] ,
' ^vsprojects/ ' : [ _WINDOWS_TEST_SUITE ] ,
' binding \ .gyp$ ' : [ _NODE_TEST_SUITE ] ,
' composer \ .json$ ' : [ _PHP_TEST_SUITE ] ,
' config \ .m4$ ' : [ _PHP_TEST_SUITE ] ,
' CONTRIBUTING \ .md$ ' : [ ] ,
' Gemfile$ ' : [ _RUBY_TEST_SUITE ] ,
' grpc.def$ ' : [ _WINDOWS_TEST_SUITE ] ,
' grpc \ .gemspec$ ' : [ _RUBY_TEST_SUITE ] ,
' gRPC \ .podspec$ ' : [ _OBJC_TEST_SUITE ] ,
' gRPC \ -Core \ .podspec$ ' : [ _OBJC_TEST_SUITE ] ,
' gRPC \ -ProtoRPC \ .podspec$ ' : [ _OBJC_TEST_SUITE ] ,
' gRPC \ -RxLibrary \ .podspec$ ' : [ _OBJC_TEST_SUITE ] ,
' INSTALL \ .md$ ' : [ ] ,
' LICENSE$ ' : [ ] ,
' MANIFEST \ .md$ ' : [ ] ,
' package \ .json$ ' : [ _PHP_TEST_SUITE ] ,
' package \ .xml$ ' : [ _PHP_TEST_SUITE ] ,
' PATENTS$ ' : [ ] ,
' PYTHON \ -MANIFEST \ .in$ ' : [ _PYTHON_TEST_SUITE ] ,
' README \ .md$ ' : [ ] ,
' requirements \ .txt$ ' : [ _PYTHON_TEST_SUITE ] ,
' setup \ .cfg$ ' : [ _PYTHON_TEST_SUITE ] ,
' setup \ .py$ ' : [ _PYTHON_TEST_SUITE ]
}
# Add all triggers to their respective test suites
for trigger , test_suites in _WHITELIST_DICT . iteritems ( ) :
for test_suite in test_suites :
@ -106,10 +133,6 @@ def _get_changed_files(base_branch):
"""
Get list of changed files between current branch and base of target merge branch
"""
# git fetch might need to be called on Jenkins slave
# todo(mattkwong): remove or uncomment below after seeing if Jenkins needs this
# call(['git', 'fetch'])
# Get file changes between branch and merge-base of specified branch
# Not combined to be Windows friendly
base_commit = check_output ( [ " git " , " merge-base " , base_branch , " HEAD " ] ) . rstrip ( )
@ -129,27 +152,17 @@ def _can_skip_tests(file_names, triggers):
return True
def _remove_irrelevant_tests ( tests , tag ) :
def _remove_irrelevant_tests ( tests , skippable_labels ) :
"""
Filters out tests by config or language - will not remove sanitizer tests
: param tests : list of all tests generated by run_tests_matrix . py
: param tag : string representing language or config to filter - " _(language)_ " or " _(config) "
: return : list of relevant tests
"""
# todo(mattkwong): find a more reliable way to filter tests - don't use shortname
return [ test for test in tests if tag not in test . shortname or
any ( san_tag in test . shortname for san_tag in [ ' _asan ' , ' _tsan ' , ' _msan ' ] ) ]
def _remove_sanitizer_tests ( tests ) :
"""
Filters out sanitizer tests
: param tests : list of all tests generated by run_tests_matrix . py
: param skippable_labels : list of languages and platforms with skippable tests
: return : list of relevant tests
"""
# todo(mattkwong): find a more reliable way to filter tests - don't use shortname
return [ test for test in tests if
all ( san_tag not in test . shortname for san_tag in [ ' _asan ' , ' _tsan ' , ' _msan ' ] ) ]
# test.labels[0] is platform and test.labels[2] is language
# We skip a test if both are considered safe to skip
return [ test for test in tests if test . labels [ 0 ] not in skippable_labels or \
test . labels [ 2 ] not in skippable_labels ]
def filter_tests ( tests , base_branch ) :
@ -158,7 +171,7 @@ def filter_tests(tests, base_branch):
: param tests : list of all tests generated by run_tests_matrix . py
: return : list of relevant tests
"""
print ( " Finding file differences between %s repo and current branch ... \n " % base_branch )
print ( " Finding file differences between gRPC %s branch and pull request ... \n " % base_branch )
changed_files = _get_changed_files ( base_branch )
for changed_file in changed_files :
print ( changed_file )
@ -170,15 +183,13 @@ def filter_tests(tests, base_branch):
for changed_file in changed_files :
if not re . match ( all_triggers , changed_file ) :
return ( tests )
# Filter out tests by language
for test_suite in _all_test_suites :
# Figure out which language and platform tests to run
skippable_labels = [ ]
for test_suite in _ALL_TEST_SUITES :
if _can_skip_tests ( changed_files , test_suite . triggers ) :
for tag in test_suite . tags :
print ( " Filtering %s tests " % tag )
tests = _remove_irrelevant_tests ( tests , tag )
# Sanitizer tests skipped if core and c++ are skipped
if _can_skip_tests ( changed_files , _cpp_test_suite . triggers + _core_test_suite . triggers ) :
print ( " Filtering Sanitizer tests " )
tests = _remove_sanitizer_tests ( tests )
for label in test_suite . labels :
print ( " Filtering %s tests " % label )
skippable_labels . append ( label )
tests = _remove_irrelevant_tests ( tests , skippable_labels )
return tests
Sree Kuchibhotla
8 years ago