Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

simplify run_tests.py

Browse Source
pull/5369/head
Jan Tattermusch 9 years ago committed by Jan Tattermusch
parent 70a57e42f4
commit 77db432c2d
1 changed files with 135 additions and 122 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 257
      tools/run_tests/run_tests.py

257
tools/run_tests/run_tests.py
Unescape View File

@ -118,6 +118,11 @@ def get_c_tests(travis, test_lang) :
not (travis and tgt['flaky'])]
def _check_compiler(compiler, supported_compilers):
if compiler not in supported_compilers:
raise Exception('Compiler %s not supported.' % compiler)
class CLanguage(object):
def __init__(self, make_target, test_lang):
@ -125,38 +130,50 @@ class CLanguage(object):
self.platform = platform_string()
self.test_lang = test_lang
def test_specs(self, config, args):
def configure(self, config, args):
self.config = config
self.args = args
if self.platform == 'windows':
_check_compiler(self.args.compiler, ['default',
'vs2010',
'vs2013',
'vs2015'])
else:
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
out = []
binaries = get_c_tests(args.travis, self.test_lang)
binaries = get_c_tests(self.args.travis, self.test_lang)
for target in binaries:
if config.build_config in target['exclude_configs']:
if self.config.build_config in target['exclude_configs']:
continue
if self.platform == 'windows':
binary = 'vsprojects/%s%s/%s.exe' % (
'x64/' if args.arch == 'x64' else '',
_WINDOWS_CONFIG[config.build_config],
'x64/' if self.args.arch == 'x64' else '',
_WINDOWS_CONFIG[self.config.build_config],
target['name'])
else:
binary = 'bins/%s/%s' % (config.build_config, target['name'])
binary = 'bins/%s/%s' % (self.config.build_config, target['name'])
if os.path.isfile(binary):
cmdline = [binary] + target['args']
out.append(config.job_spec(cmdline, [binary],
shortname=' '.join(cmdline),
cpu_cost=target['cpu_cost'],
environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
os.path.abspath(os.path.dirname(
sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')}))
elif args.regex == '.*' or platform_string() == 'windows':
out.append(self.config.job_spec(cmdline, [binary],
shortname=' '.join(cmdline),
cpu_cost=target['cpu_cost'],
environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
os.path.abspath(os.path.dirname(
sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')}))
elif self.args.regex == '.*' or self.platform == 'windows':
print '\nWARNING: binary not found, skipping', binary
return sorted(out)
def make_targets(self, test_regex):
if platform_string() != 'windows' and test_regex != '.*':
def make_targets(self):
test_regex = self.args.regex
if self.platform != 'windows' and self.args.regex != '.*':
# use the regex to minimize the number of things to build
return [os.path.basename(target['name'])
for target in get_c_tests(False, self.test_lang)
if re.search(test_regex, '/' + target['name'])]
if platform_string() == 'windows':
if self.platform == 'windows':
# don't build tools on windows just yet
return ['buildtests_%s' % self.make_target]
return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target]
@ -182,11 +199,8 @@ class CLanguage(object):
def makefile_name(self):
return 'Makefile'
def supports_multi_config(self):
return True
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(arch)
def dockerfile_dir(self):
return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return self.make_target
@ -198,13 +212,18 @@ class NodeLanguage(object):
self.platform = platform_string()
self.node_version = '0.12'
def test_specs(self, config, args):
def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
if self.platform == 'windows':
return [config.job_spec(['tools\\run_tests\\run_node.bat'], None)]
return [self.config.job_spec(['tools\\run_tests\\run_node.bat'], None)]
else:
return [config.job_spec(['tools/run_tests/run_node.sh', self.node_version],
None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
return [self.config.job_spec(['tools/run_tests/run_node.sh', self.node_version],
None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self):
if self.platform == 'windows':
@ -212,7 +231,7 @@ class NodeLanguage(object):
else:
return [['tools/run_tests/pre_build_node.sh', self.node_version]]
def make_targets(self, test_regex):
def make_targets(self):
return []
def make_options(self):
@ -230,11 +249,8 @@ class NodeLanguage(object):
def makefile_name(self):
return 'Makefile'
def supports_multi_config(self):
return False
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(arch)
def dockerfile_dir(self):
return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'node'
@ -242,14 +258,19 @@ class NodeLanguage(object):
class PhpLanguage(object):
def test_specs(self, config, args):
return [config.job_spec(['src/php/bin/run_tests.sh'], None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
return [self.config.job_spec(['src/php/bin/run_tests.sh'], None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self):
return []
def make_targets(self, test_regex):
def make_targets(self):
return ['static_c', 'shared_c']
def make_options(self):
@ -264,11 +285,8 @@ class PhpLanguage(object):
def makefile_name(self):
return 'Makefile'
def supports_multi_config(self):
return False
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(arch)
def dockerfile_dir(self):
return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'php'
@ -280,10 +298,15 @@ class PythonLanguage(object):
self._build_python_versions = ['2.7']
self._has_python_versions = []
def test_specs(self, config, args):
def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
environment['PYVER'] = '2.7'
return [config.job_spec(
return [self.config.job_spec(
['tools/run_tests/run_python.sh'],
None,
environ=environment,
@ -294,7 +317,7 @@ class PythonLanguage(object):
def pre_build_steps(self):
return []
def make_targets(self, test_regex):
def make_targets(self):
return ['static_c', 'grpc_python_plugin', 'shared_c']
def make_options(self):
@ -320,11 +343,8 @@ class PythonLanguage(object):
def makefile_name(self):
return 'Makefile'
def supports_multi_config(self):
return False
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(arch)
def dockerfile_dir(self):
return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'python'
@ -332,10 +352,15 @@ class PythonLanguage(object):
class RubyLanguage(object):
def test_specs(self, config, args):
return [config.job_spec(['tools/run_tests/run_ruby.sh'], None,
timeout_seconds=10*60,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], None,
timeout_seconds=10*60,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self):
return [['tools/run_tests/pre_build_ruby.sh']]
@ -355,27 +380,30 @@ class RubyLanguage(object):
def makefile_name(self):
return 'Makefile'
def supports_multi_config(self):
return False
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(arch)
def dockerfile_dir(self):
return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'ruby'
class CSharpLanguage(object):
def __init__(self):
self.platform = platform_string()
def test_specs(self, config, args):
def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
with open('src/csharp/tests.json') as f:
tests_json = json.load(f)
assemblies = tests_json['assemblies']
tests = tests_json['tests']
msbuild_config = _WINDOWS_CONFIG[config.build_config]
msbuild_config = _WINDOWS_CONFIG[self.config.build_config]
assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a)
for a in assemblies]
@ -387,13 +415,13 @@ class CSharpLanguage(object):
else:
script_name = 'tools/run_tests/run_csharp.sh'
if config.build_config == 'gcov':
if self.config.build_config == 'gcov':
# On Windows, we only collect C# code coverage.
# On Linux, we only collect coverage for native extension.
# For code coverage all tests need to run as one suite.
return [config.job_spec([script_name] + extra_args, None,
shortname='csharp.coverage',
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
return [self.config.job_spec([script_name] + extra_args, None,
shortname='csharp.coverage',
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
else:
specs = []
for test in tests:
@ -402,9 +430,9 @@ class CSharpLanguage(object):
# use different output directory for each test to prevent
# TestResult.xml clash between parallel test runs.
cmdline += ['-work=test-result/%s' % uuid.uuid4()]
specs.append(config.job_spec(cmdline, None,
shortname='csharp.%s' % test,
environ=_FORCE_ENVIRON_FOR_WRAPPERS))
specs.append(self.config.job_spec(cmdline, None,
shortname='csharp.%s' % test,
environ=_FORCE_ENVIRON_FOR_WRAPPERS))
return specs
def pre_build_steps(self):
@ -413,7 +441,7 @@ class CSharpLanguage(object):
else:
return [['tools/run_tests/pre_build_csharp.sh']]
def make_targets(self, test_regex):
def make_targets(self):
# For Windows, this target doesn't really build anything,
# everything is build by buildall script later.
if self.platform == 'windows':
@ -440,11 +468,8 @@ class CSharpLanguage(object):
def makefile_name(self):
return 'Makefile'
def supports_multi_config(self):
return False
def dockerfile_dir(self, config, arch):
return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(arch)
def dockerfile_dir(self):
return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'csharp'
@ -452,14 +477,19 @@ class CSharpLanguage(object):
class ObjCLanguage(object):
def test_specs(self, config, args):
return [config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
return [self.config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self):
return []
def make_targets(self, test_regex):
def make_targets(self):
return ['grpc_objective_c_plugin', 'interop_server']
def make_options(self):
@ -474,10 +504,7 @@ class ObjCLanguage(object):
def makefile_name(self):
return 'Makefile'
def supports_multi_config(self):
return False
def dockerfile_dir(self, config, arch):
def dockerfile_dir(self):
return None
def __str__(self):
@ -486,18 +513,23 @@ class ObjCLanguage(object):
class Sanity(object):
def test_specs(self, config, args):
def configure(self, config, args):
self.config = config
self.args = args
_check_compiler(self.args.compiler, ['default'])
def test_specs(self):
import yaml
with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f:
return [config.job_spec(cmd['script'].split(), None,
timeout_seconds=None, environ={'TEST': 'true'},
cpu_cost=cmd.get('cpu_cost', 1))
return [self.config.job_spec(cmd['script'].split(), None,
timeout_seconds=None, environ={'TEST': 'true'},
cpu_cost=cmd.get('cpu_cost', 1))
for cmd in yaml.load(f)]
def pre_build_steps(self):
return []
def make_targets(self, test_regex):
def make_targets(self):
return ['run_dep_checks']
def make_options(self):
@ -512,10 +544,7 @@ class Sanity(object):
def makefile_name(self):
return 'Makefile'
def supports_multi_config(self):
return False
def dockerfile_dir(self, config, arch):
def dockerfile_dir(self):
return 'tools/dockerfile/test/sanity'
def __str__(self):
@ -527,7 +556,6 @@ with open('tools/run_tests/configs.json') as f:
_CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read()))
_DEFAULT = ['opt']
_LANGUAGES = {
'c++': CLanguage('cxx', 'c++'),
'c': CLanguage('c', 'c'),
@ -540,6 +568,7 @@ _LANGUAGES = {
'sanity': Sanity()
}
_WINDOWS_CONFIG = {
'dbg': 'Debug',
'opt': 'Release',
@ -648,9 +677,8 @@ def runs_per_test_type(arg_str):
# parse command line
argp = argparse.ArgumentParser(description='Run grpc tests.')
argp.add_argument('-c', '--config',
choices=['all'] + sorted(_CONFIGS.keys()),
nargs='+',
default=_DEFAULT)
choices=sorted(_CONFIGS.keys()),
default='opt')
argp.add_argument('-n', '--runs_per_test', default=1, type=runs_per_test_type,
help='A positive integer or "inf". If "inf", all tests will run in an '
'infinite loop. Especially useful in combination with "-f"')
@ -696,7 +724,7 @@ argp.add_argument('--arch',
argp.add_argument('--compiler',
choices=['default', 'vs2010', 'vs2013', 'vs2015'],
default='default',
help='Selects compiler to use. For some platforms "default" is the only supported choice.')
help='Selects compiler to use. Allowed values depend on the platform and language.')
argp.add_argument('--build_only',
default=False,
action='store_const',
@ -742,11 +770,8 @@ if need_to_regenerate_projects:
# grab config
run_configs = set(_CONFIGS[cfg]
for cfg in itertools.chain.from_iterable(
_CONFIGS.iterkeys() if x == 'all' else [x]
for x in args.config))
build_configs = set(cfg.build_config for cfg in run_configs)
run_config = _CONFIGS[args.config]
build_config = run_config.build_config
if args.travis:
_FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}
@ -762,12 +787,8 @@ if 'gcov' in args.config:
lang_list.remove(bad)
languages = set(_LANGUAGES[l] for l in lang_list)
if len(build_configs) > 1:
for language in languages:
if not language.supports_multi_config():
print language, 'does not support multiple build configurations'
sys.exit(1)
for l in languages:
l.configure(run_config, args)
language_make_options=[]
if any(language.make_options() for language in languages):
@ -777,8 +798,8 @@ if any(language.make_options() for language in languages):
else:
language_make_options = next(iter(languages)).make_options()
if len(languages) != 1 or len(build_configs) != 1:
print 'Multi-language and multi-config testing is not supported.'
if len(languages) != 1:
print 'Multi-language testing is not supported.'
sys.exit(1)
if args.use_docker:
@ -809,10 +830,6 @@ if args.use_docker:
env=env)
sys.exit(0)
if platform_string() != 'windows' and args.compiler != 'default':
print 'Compiler %s not supported on current platform.' % args.compiler
sys.exit(1)
_check_arch_option(args.arch)
def make_jobspec(cfg, targets, makefile='Makefile'):
@ -852,7 +869,7 @@ make_targets = {}
for l in languages:
makefile = l.makefile_name()
make_targets[makefile] = make_targets.get(makefile, set()).union(
set(l.make_targets(args.regex)))
set(l.make_targets()))
def build_step_environ(cfg):
environ = {'CONFIG': cfg}
@ -862,22 +879,19 @@ def build_step_environ(cfg):
return environ
build_steps = list(set(
jobset.JobSpec(cmdline, environ=build_step_environ(cfg), flake_retries=5)
for cfg in build_configs
jobset.JobSpec(cmdline, environ=build_step_environ(build_config), flake_retries=5)
for l in languages
for cmdline in l.pre_build_steps()))
if make_targets:
make_commands = itertools.chain.from_iterable(make_jobspec(cfg, list(targets), makefile) for cfg in build_configs for (makefile, targets) in make_targets.iteritems())
make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems())
build_steps.extend(set(make_commands))
build_steps.extend(set(
jobset.JobSpec(cmdline, environ=build_step_environ(cfg), timeout_seconds=None)
for cfg in build_configs
jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None)
for l in languages
for cmdline in l.build_steps()))
post_tests_steps = list(set(
jobset.JobSpec(cmdline, environ=build_step_environ(cfg))
for cfg in build_configs
jobset.JobSpec(cmdline, environ=build_step_environ(build_config))
for l in languages
for cmdline in l.post_tests_steps()))
runs_per_test = args.runs_per_test
@ -1068,9 +1082,8 @@ def _build_and_run(
infinite_runs = runs_per_test == 0
one_run = set(
spec
for config in run_configs
for language in languages
for spec in language.test_specs(config, args)
for spec in language.test_specs()
if re.search(args.regex, spec.shortname))
# When running on travis, we want out test runs to be as similar as possible
# for reproducibility purposes.

Write Preview
Loading…
Cancel
Save