Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

simplify run_tests.py

Browse Source
pull/5369/head
Jan Tattermusch 9 years ago committed by Jan Tattermusch
parent 70a57e42f4
commit 77db432c2d
1 changed files with 135 additions and 122 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 257
      tools/run_tests/run_tests.py

257
tools/run_tests/run_tests.py
Unescape View File

@ -118,6 +118,11 @@ def get_c_tests(travis, test_lang) :
not (travis and tgt['flaky'])] not (travis and tgt['flaky'])]
def _check_compiler(compiler, supported_compilers):
if compiler not in supported_compilers:
raise Exception('Compiler %s not supported.' % compiler)
class CLanguage(object): class CLanguage(object):
def __init__(self, make_target, test_lang): def __init__(self, make_target, test_lang):
@ -125,38 +130,50 @@ class CLanguage(object):
self.platform = platform_string() self.platform = platform_string()
self.test_lang = test_lang self.test_lang = test_lang
def test_specs(self, config, args): def configure(self, config, args):
self.config = config
self.args = args
if self.platform == 'windows':
_check_compiler(self.args.compiler, ['default',
'vs2010',
'vs2013',
'vs2015'])
else:
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
out = [] out = []
binaries = get_c_tests(args.travis, self.test_lang) binaries = get_c_tests(self.args.travis, self.test_lang)
for target in binaries: for target in binaries:
if config.build_config in target['exclude_configs']: if self.config.build_config in target['exclude_configs']:
continue continue
if self.platform == 'windows': if self.platform == 'windows':
binary = 'vsprojects/%s%s/%s.exe' % ( binary = 'vsprojects/%s%s/%s.exe' % (
'x64/' if args.arch == 'x64' else '', 'x64/' if self.args.arch == 'x64' else '',
_WINDOWS_CONFIG[config.build_config], _WINDOWS_CONFIG[self.config.build_config],
target['name']) target['name'])
else: else:
binary = 'bins/%s/%s' % (config.build_config, target['name']) binary = 'bins/%s/%s' % (self.config.build_config, target['name'])
if os.path.isfile(binary): if os.path.isfile(binary):
cmdline = [binary] + target['args'] cmdline = [binary] + target['args']
out.append(config.job_spec(cmdline, [binary], out.append(self.config.job_spec(cmdline, [binary],
shortname=' '.join(cmdline), shortname=' '.join(cmdline),
cpu_cost=target['cpu_cost'], cpu_cost=target['cpu_cost'],
environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
os.path.abspath(os.path.dirname( os.path.abspath(os.path.dirname(
sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')})) sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')}))
elif args.regex == '.*' or platform_string() == 'windows': elif self.args.regex == '.*' or self.platform == 'windows':
print '\nWARNING: binary not found, skipping', binary print '\nWARNING: binary not found, skipping', binary
return sorted(out) return sorted(out)
def make_targets(self, test_regex): def make_targets(self):
if platform_string() != 'windows' and test_regex != '.*': test_regex = self.args.regex
if self.platform != 'windows' and self.args.regex != '.*':
# use the regex to minimize the number of things to build # use the regex to minimize the number of things to build
return [os.path.basename(target['name']) return [os.path.basename(target['name'])
for target in get_c_tests(False, self.test_lang) for target in get_c_tests(False, self.test_lang)
if re.search(test_regex, '/' + target['name'])] if re.search(test_regex, '/' + target['name'])]
if platform_string() == 'windows': if self.platform == 'windows':
# don't build tools on windows just yet # don't build tools on windows just yet
return ['buildtests_%s' % self.make_target] return ['buildtests_%s' % self.make_target]
return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target] return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target]
@ -182,11 +199,8 @@ class CLanguage(object):
def makefile_name(self): def makefile_name(self):
return 'Makefile' return 'Makefile'
def supports_multi_config(self): def dockerfile_dir(self):
return True return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(self.args.arch)
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(arch)
def __str__(self): def __str__(self):
return self.make_target return self.make_target
@ -198,13 +212,18 @@ class NodeLanguage(object):
self.platform = platform_string() self.platform = platform_string()
self.node_version = '0.12' self.node_version = '0.12'
def test_specs(self, config, args): def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
if self.platform == 'windows': if self.platform == 'windows':
return [config.job_spec(['tools\\run_tests\\run_node.bat'], None)] return [self.config.job_spec(['tools\\run_tests\\run_node.bat'], None)]
else: else:
return [config.job_spec(['tools/run_tests/run_node.sh', self.node_version], return [self.config.job_spec(['tools/run_tests/run_node.sh', self.node_version],
None, None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)] environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self): def pre_build_steps(self):
if self.platform == 'windows': if self.platform == 'windows':
@ -212,7 +231,7 @@ class NodeLanguage(object):
else: else:
return [['tools/run_tests/pre_build_node.sh', self.node_version]] return [['tools/run_tests/pre_build_node.sh', self.node_version]]
def make_targets(self, test_regex): def make_targets(self):
return [] return []
def make_options(self): def make_options(self):
@ -230,11 +249,8 @@ class NodeLanguage(object):
def makefile_name(self): def makefile_name(self):
return 'Makefile' return 'Makefile'
def supports_multi_config(self): def dockerfile_dir(self):
return False return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch)
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(arch)
def __str__(self): def __str__(self):
return 'node' return 'node'
@ -242,14 +258,19 @@ class NodeLanguage(object):
class PhpLanguage(object): class PhpLanguage(object):
def test_specs(self, config, args): def configure(self, config, args):
return [config.job_spec(['src/php/bin/run_tests.sh'], None, self.config = config
environ=_FORCE_ENVIRON_FOR_WRAPPERS)] self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
return [self.config.job_spec(['src/php/bin/run_tests.sh'], None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self): def pre_build_steps(self):
return [] return []
def make_targets(self, test_regex): def make_targets(self):
return ['static_c', 'shared_c'] return ['static_c', 'shared_c']
def make_options(self): def make_options(self):
@ -264,11 +285,8 @@ class PhpLanguage(object):
def makefile_name(self): def makefile_name(self):
return 'Makefile' return 'Makefile'
def supports_multi_config(self): def dockerfile_dir(self):
return False return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(self.args.arch)
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(arch)
def __str__(self): def __str__(self):
return 'php' return 'php'
@ -280,10 +298,15 @@ class PythonLanguage(object):
self._build_python_versions = ['2.7'] self._build_python_versions = ['2.7']
self._has_python_versions = [] self._has_python_versions = []
def test_specs(self, config, args): def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
environment['PYVER'] = '2.7' environment['PYVER'] = '2.7'
return [config.job_spec( return [self.config.job_spec(
['tools/run_tests/run_python.sh'], ['tools/run_tests/run_python.sh'],
None, None,
environ=environment, environ=environment,
@ -294,7 +317,7 @@ class PythonLanguage(object):
def pre_build_steps(self): def pre_build_steps(self):
return [] return []
def make_targets(self, test_regex): def make_targets(self):
return ['static_c', 'grpc_python_plugin', 'shared_c'] return ['static_c', 'grpc_python_plugin', 'shared_c']
def make_options(self): def make_options(self):
@ -320,11 +343,8 @@ class PythonLanguage(object):
def makefile_name(self): def makefile_name(self):
return 'Makefile' return 'Makefile'
def supports_multi_config(self): def dockerfile_dir(self):
return False return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch)
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(arch)
def __str__(self): def __str__(self):
return 'python' return 'python'
@ -332,10 +352,15 @@ class PythonLanguage(object):
class RubyLanguage(object): class RubyLanguage(object):
def test_specs(self, config, args): def configure(self, config, args):
return [config.job_spec(['tools/run_tests/run_ruby.sh'], None, self.config = config
timeout_seconds=10*60, self.args = args
environ=_FORCE_ENVIRON_FOR_WRAPPERS)] _check_compiler(self.args.compiler, ['default'])
def test_specs(self):
return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], None,
timeout_seconds=10*60,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self): def pre_build_steps(self):
return [['tools/run_tests/pre_build_ruby.sh']] return [['tools/run_tests/pre_build_ruby.sh']]
@ -355,27 +380,30 @@ class RubyLanguage(object):
def makefile_name(self): def makefile_name(self):
return 'Makefile' return 'Makefile'
def supports_multi_config(self): def dockerfile_dir(self):
return False return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(self.args.arch)
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(arch)
def __str__(self): def __str__(self):
return 'ruby' return 'ruby'
class CSharpLanguage(object): class CSharpLanguage(object):
def __init__(self): def __init__(self):
self.platform = platform_string() self.platform = platform_string()
def test_specs(self, config, args): def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
with open('src/csharp/tests.json') as f: with open('src/csharp/tests.json') as f:
tests_json = json.load(f) tests_json = json.load(f)
assemblies = tests_json['assemblies'] assemblies = tests_json['assemblies']
tests = tests_json['tests'] tests = tests_json['tests']
msbuild_config = _WINDOWS_CONFIG[config.build_config] msbuild_config = _WINDOWS_CONFIG[self.config.build_config]
assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a) assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a)
for a in assemblies] for a in assemblies]
@ -387,13 +415,13 @@ class CSharpLanguage(object):
else: else:
script_name = 'tools/run_tests/run_csharp.sh' script_name = 'tools/run_tests/run_csharp.sh'
if config.build_config == 'gcov': if self.config.build_config == 'gcov':
# On Windows, we only collect C# code coverage. # On Windows, we only collect C# code coverage.
# On Linux, we only collect coverage for native extension. # On Linux, we only collect coverage for native extension.
# For code coverage all tests need to run as one suite. # For code coverage all tests need to run as one suite.
return [config.job_spec([script_name] + extra_args, None, return [self.config.job_spec([script_name] + extra_args, None,
shortname='csharp.coverage', shortname='csharp.coverage',
environ=_FORCE_ENVIRON_FOR_WRAPPERS)] environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
else: else:
specs = [] specs = []
for test in tests: for test in tests:
@ -402,9 +430,9 @@ class CSharpLanguage(object):
# use different output directory for each test to prevent # use different output directory for each test to prevent
# TestResult.xml clash between parallel test runs. # TestResult.xml clash between parallel test runs.
cmdline += ['-work=test-result/%s' % uuid.uuid4()] cmdline += ['-work=test-result/%s' % uuid.uuid4()]
specs.append(config.job_spec(cmdline, None, specs.append(self.config.job_spec(cmdline, None,
shortname='csharp.%s' % test, shortname='csharp.%s' % test,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)) environ=_FORCE_ENVIRON_FOR_WRAPPERS))
return specs return specs
def pre_build_steps(self): def pre_build_steps(self):
@ -413,7 +441,7 @@ class CSharpLanguage(object):
else: else:
return [['tools/run_tests/pre_build_csharp.sh']] return [['tools/run_tests/pre_build_csharp.sh']]
def make_targets(self, test_regex): def make_targets(self):
# For Windows, this target doesn't really build anything, # For Windows, this target doesn't really build anything,
# everything is build by buildall script later. # everything is build by buildall script later.
if self.platform == 'windows': if self.platform == 'windows':
@ -440,11 +468,8 @@ class CSharpLanguage(object):
def makefile_name(self): def makefile_name(self):
return 'Makefile' return 'Makefile'
def supports_multi_config(self): def dockerfile_dir(self):
return False return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(self.args.arch)
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(arch)
def __str__(self): def __str__(self):
return 'csharp' return 'csharp'
@ -452,14 +477,19 @@ class CSharpLanguage(object):
class ObjCLanguage(object): class ObjCLanguage(object):
def test_specs(self, config, args): def configure(self, config, args):
return [config.job_spec(['src/objective-c/tests/run_tests.sh'], None, self.config = config
environ=_FORCE_ENVIRON_FOR_WRAPPERS)] self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
return [self.config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self): def pre_build_steps(self):
return [] return []
def make_targets(self, test_regex): def make_targets(self):
return ['grpc_objective_c_plugin', 'interop_server'] return ['grpc_objective_c_plugin', 'interop_server']
def make_options(self): def make_options(self):
@ -474,10 +504,7 @@ class ObjCLanguage(object):
def makefile_name(self): def makefile_name(self):
return 'Makefile' return 'Makefile'
def supports_multi_config(self): def dockerfile_dir(self):
return False
def dockerfile_dir(self, config, arch):
return None return None
def __str__(self): def __str__(self):
@ -486,18 +513,23 @@ class ObjCLanguage(object):
class Sanity(object): class Sanity(object):
def test_specs(self, config, args): def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
import yaml import yaml
with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f: with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f:
return [config.job_spec(cmd['script'].split(), None, return [self.config.job_spec(cmd['script'].split(), None,
timeout_seconds=None, environ={'TEST': 'true'}, timeout_seconds=None, environ={'TEST': 'true'},
cpu_cost=cmd.get('cpu_cost', 1)) cpu_cost=cmd.get('cpu_cost', 1))
for cmd in yaml.load(f)] for cmd in yaml.load(f)]
def pre_build_steps(self): def pre_build_steps(self):
return [] return []
def make_targets(self, test_regex): def make_targets(self):
return ['run_dep_checks'] return ['run_dep_checks']
def make_options(self): def make_options(self):
@ -512,10 +544,7 @@ class Sanity(object):
def makefile_name(self): def makefile_name(self):
return 'Makefile' return 'Makefile'
def supports_multi_config(self): def dockerfile_dir(self):
return False
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/sanity' return 'tools/dockerfile/test/sanity'
def __str__(self): def __str__(self):
@ -527,7 +556,6 @@ with open('tools/run_tests/configs.json') as f:
_CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read())) _CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read()))
_DEFAULT = ['opt']
_LANGUAGES = { _LANGUAGES = {
'c++': CLanguage('cxx', 'c++'), 'c++': CLanguage('cxx', 'c++'),
'c': CLanguage('c', 'c'), 'c': CLanguage('c', 'c'),
@ -540,6 +568,7 @@ _LANGUAGES = {
'sanity': Sanity() 'sanity': Sanity()
} }
_WINDOWS_CONFIG = { _WINDOWS_CONFIG = {
'dbg': 'Debug', 'dbg': 'Debug',
'opt': 'Release', 'opt': 'Release',
@ -648,9 +677,8 @@ def runs_per_test_type(arg_str):
# parse command line # parse command line
argp = argparse.ArgumentParser(description='Run grpc tests.') argp = argparse.ArgumentParser(description='Run grpc tests.')
argp.add_argument('-c', '--config', argp.add_argument('-c', '--config',
choices=['all'] + sorted(_CONFIGS.keys()), choices=sorted(_CONFIGS.keys()),
nargs='+', default='opt')
default=_DEFAULT)
argp.add_argument('-n', '--runs_per_test', default=1, type=runs_per_test_type, argp.add_argument('-n', '--runs_per_test', default=1, type=runs_per_test_type,
help='A positive integer or "inf". If "inf", all tests will run in an ' help='A positive integer or "inf". If "inf", all tests will run in an '
'infinite loop. Especially useful in combination with "-f"') 'infinite loop. Especially useful in combination with "-f"')
@ -696,7 +724,7 @@ argp.add_argument('--arch',
argp.add_argument('--compiler', argp.add_argument('--compiler',
choices=['default', 'vs2010', 'vs2013', 'vs2015'], choices=['default', 'vs2010', 'vs2013', 'vs2015'],
default='default', default='default',
help='Selects compiler to use. For some platforms "default" is the only supported choice.') help='Selects compiler to use. Allowed values depend on the platform and language.')
argp.add_argument('--build_only', argp.add_argument('--build_only',
default=False, default=False,
action='store_const', action='store_const',
@ -742,11 +770,8 @@ if need_to_regenerate_projects:
# grab config # grab config
run_configs = set(_CONFIGS[cfg] run_config = _CONFIGS[args.config]
for cfg in itertools.chain.from_iterable( build_config = run_config.build_config
_CONFIGS.iterkeys() if x == 'all' else [x]
for x in args.config))
build_configs = set(cfg.build_config for cfg in run_configs)
if args.travis: if args.travis:
_FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'} _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}
@ -762,12 +787,8 @@ if 'gcov' in args.config:
lang_list.remove(bad) lang_list.remove(bad)
languages = set(_LANGUAGES[l] for l in lang_list) languages = set(_LANGUAGES[l] for l in lang_list)
for l in languages:
if len(build_configs) > 1: l.configure(run_config, args)
for language in languages:
if not language.supports_multi_config():
print language, 'does not support multiple build configurations'
sys.exit(1)
language_make_options=[] language_make_options=[]
if any(language.make_options() for language in languages): if any(language.make_options() for language in languages):
@ -777,8 +798,8 @@ if any(language.make_options() for language in languages):
else: else:
language_make_options = next(iter(languages)).make_options() language_make_options = next(iter(languages)).make_options()
if len(languages) != 1 or len(build_configs) != 1: if len(languages) != 1:
print 'Multi-language and multi-config testing is not supported.' print 'Multi-language testing is not supported.'
sys.exit(1) sys.exit(1)
if args.use_docker: if args.use_docker:
@ -809,10 +830,6 @@ if args.use_docker:
env=env) env=env)
sys.exit(0) sys.exit(0)
if platform_string() != 'windows' and args.compiler != 'default':
print 'Compiler %s not supported on current platform.' % args.compiler
sys.exit(1)
_check_arch_option(args.arch) _check_arch_option(args.arch)
def make_jobspec(cfg, targets, makefile='Makefile'): def make_jobspec(cfg, targets, makefile='Makefile'):
@ -852,7 +869,7 @@ make_targets = {}
for l in languages: for l in languages:
makefile = l.makefile_name() makefile = l.makefile_name()
make_targets[makefile] = make_targets.get(makefile, set()).union( make_targets[makefile] = make_targets.get(makefile, set()).union(
set(l.make_targets(args.regex))) set(l.make_targets()))
def build_step_environ(cfg): def build_step_environ(cfg):
environ = {'CONFIG': cfg} environ = {'CONFIG': cfg}
@ -862,22 +879,19 @@ def build_step_environ(cfg):
return environ return environ
build_steps = list(set( build_steps = list(set(
jobset.JobSpec(cmdline, environ=build_step_environ(cfg), flake_retries=5) jobset.JobSpec(cmdline, environ=build_step_environ(build_config), flake_retries=5)
for cfg in build_configs
for l in languages for l in languages
for cmdline in l.pre_build_steps())) for cmdline in l.pre_build_steps()))
if make_targets: if make_targets:
make_commands = itertools.chain.from_iterable(make_jobspec(cfg, list(targets), makefile) for cfg in build_configs for (makefile, targets) in make_targets.iteritems()) make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems())
build_steps.extend(set(make_commands)) build_steps.extend(set(make_commands))
build_steps.extend(set( build_steps.extend(set(
jobset.JobSpec(cmdline, environ=build_step_environ(cfg), timeout_seconds=None) jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None)
for cfg in build_configs
for l in languages for l in languages
for cmdline in l.build_steps())) for cmdline in l.build_steps()))
post_tests_steps = list(set( post_tests_steps = list(set(
jobset.JobSpec(cmdline, environ=build_step_environ(cfg)) jobset.JobSpec(cmdline, environ=build_step_environ(build_config))
for cfg in build_configs
for l in languages for l in languages
for cmdline in l.post_tests_steps())) for cmdline in l.post_tests_steps()))
runs_per_test = args.runs_per_test runs_per_test = args.runs_per_test
@ -1068,9 +1082,8 @@ def _build_and_run(
infinite_runs = runs_per_test == 0 infinite_runs = runs_per_test == 0
one_run = set( one_run = set(
spec spec
for config in run_configs
for language in languages for language in languages
for spec in language.test_specs(config, args) for spec in language.test_specs()
if re.search(args.regex, spec.shortname)) if re.search(args.regex, spec.shortname))
# When running on travis, we want out test runs to be as similar as possible # When running on travis, we want out test runs to be as similar as possible
# for reproducibility purposes. # for reproducibility purposes.

Write Preview
Loading…
Cancel
Save