|
|
|
@ -30,6 +30,8 @@ |
|
|
|
|
|
|
|
|
|
|
|
"""Run tests in parallel.""" |
|
|
|
"""Run tests in parallel.""" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
from __future__ import print_function |
|
|
|
|
|
|
|
|
|
|
|
import argparse |
|
|
|
import argparse |
|
|
|
import ast |
|
|
|
import ast |
|
|
|
import collections |
|
|
|
import collections |
|
|
|
@ -48,7 +50,7 @@ import sys |
|
|
|
import tempfile |
|
|
|
import tempfile |
|
|
|
import traceback |
|
|
|
import traceback |
|
|
|
import time |
|
|
|
import time |
|
|
|
import urllib2 |
|
|
|
from six.moves import urllib |
|
|
|
import uuid |
|
|
|
import uuid |
|
|
|
|
|
|
|
|
|
|
|
import jobset |
|
|
|
import jobset |
|
|
|
@ -93,7 +95,7 @@ class Config(object): |
|
|
|
would like to run |
|
|
|
would like to run |
|
|
|
""" |
|
|
|
""" |
|
|
|
actual_environ = self.environ.copy() |
|
|
|
actual_environ = self.environ.copy() |
|
|
|
for k, v in environ.iteritems(): |
|
|
|
for k, v in environ.items(): |
|
|
|
actual_environ[k] = v |
|
|
|
actual_environ[k] = v |
|
|
|
return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, |
|
|
|
return jobset.JobSpec(cmdline=self.tool_prefix + cmdline, |
|
|
|
shortname=shortname, |
|
|
|
shortname=shortname, |
|
|
|
@ -202,7 +204,7 @@ class CLanguage(object): |
|
|
|
flaky=target.get('flaky', False), |
|
|
|
flaky=target.get('flaky', False), |
|
|
|
environ=env)) |
|
|
|
environ=env)) |
|
|
|
elif self.args.regex == '.*' or self.platform == 'windows': |
|
|
|
elif self.args.regex == '.*' or self.platform == 'windows': |
|
|
|
print '\nWARNING: binary not found, skipping', binary |
|
|
|
print('\nWARNING: binary not found, skipping', binary) |
|
|
|
return sorted(out) |
|
|
|
return sorted(out) |
|
|
|
|
|
|
|
|
|
|
|
def make_targets(self): |
|
|
|
def make_targets(self): |
|
|
|
@ -393,7 +395,7 @@ class PythonLanguage(object): |
|
|
|
return [self.config.job_spec( |
|
|
|
return [self.config.job_spec( |
|
|
|
config.run, |
|
|
|
config.run, |
|
|
|
timeout_seconds=5*60, |
|
|
|
timeout_seconds=5*60, |
|
|
|
environ=dict(environment.items() + |
|
|
|
environ=dict(list(environment.items()) + |
|
|
|
[('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]), |
|
|
|
[('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]), |
|
|
|
shortname='%s.test.%s' % (config.name, suite_name),) |
|
|
|
shortname='%s.test.%s' % (config.name, suite_name),) |
|
|
|
for suite_name in tests_json |
|
|
|
for suite_name in tests_json |
|
|
|
@ -764,7 +766,7 @@ def _windows_arch_option(arch): |
|
|
|
elif arch == 'x64': |
|
|
|
elif arch == 'x64': |
|
|
|
return '/p:Platform=x64' |
|
|
|
return '/p:Platform=x64' |
|
|
|
else: |
|
|
|
else: |
|
|
|
print 'Architecture %s not supported.' % arch |
|
|
|
print('Architecture %s not supported.' % arch) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -782,11 +784,11 @@ def _check_arch_option(arch): |
|
|
|
elif runtime_arch == '32bit' and arch == 'x86': |
|
|
|
elif runtime_arch == '32bit' and arch == 'x86': |
|
|
|
return |
|
|
|
return |
|
|
|
else: |
|
|
|
else: |
|
|
|
print 'Architecture %s does not match current runtime architecture.' % arch |
|
|
|
print('Architecture %s does not match current runtime architecture.' % arch) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
else: |
|
|
|
else: |
|
|
|
if args.arch != 'default': |
|
|
|
if args.arch != 'default': |
|
|
|
print 'Architecture %s not supported on current platform.' % args.arch |
|
|
|
print('Architecture %s not supported on current platform.' % args.arch) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -800,7 +802,7 @@ def _windows_build_bat(compiler): |
|
|
|
elif compiler == 'vs2010': |
|
|
|
elif compiler == 'vs2010': |
|
|
|
return 'vsprojects\\build_vs2010.bat' |
|
|
|
return 'vsprojects\\build_vs2010.bat' |
|
|
|
else: |
|
|
|
else: |
|
|
|
print 'Compiler %s not supported.' % compiler |
|
|
|
print('Compiler %s not supported.' % compiler) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -814,7 +816,7 @@ def _windows_toolset_option(compiler): |
|
|
|
elif compiler == 'vs2010': |
|
|
|
elif compiler == 'vs2010': |
|
|
|
return '/p:PlatformToolset=v100' |
|
|
|
return '/p:PlatformToolset=v100' |
|
|
|
else: |
|
|
|
else: |
|
|
|
print 'Compiler %s not supported.' % compiler |
|
|
|
print('Compiler %s not supported.' % compiler) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -825,7 +827,7 @@ def _docker_arch_suffix(arch): |
|
|
|
elif arch == 'x86': |
|
|
|
elif arch == 'x86': |
|
|
|
return 'x86' |
|
|
|
return 'x86' |
|
|
|
else: |
|
|
|
else: |
|
|
|
print 'Architecture %s not supported with current settings.' % arch |
|
|
|
print('Architecture %s not supported with current settings.' % arch) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -941,7 +943,7 @@ for spec in args.update_submodules: |
|
|
|
branch = spec[1] |
|
|
|
branch = spec[1] |
|
|
|
cwd = 'third_party/%s' % submodule |
|
|
|
cwd = 'third_party/%s' % submodule |
|
|
|
def git(cmd, cwd=cwd): |
|
|
|
def git(cmd, cwd=cwd): |
|
|
|
print 'in %s: git %s' % (cwd, cmd) |
|
|
|
print('in %s: git %s' % (cwd, cmd)) |
|
|
|
subprocess.check_call('git %s' % cmd, cwd=cwd, shell=True) |
|
|
|
subprocess.check_call('git %s' % cmd, cwd=cwd, shell=True) |
|
|
|
git('fetch') |
|
|
|
git('fetch') |
|
|
|
git('checkout %s' % branch) |
|
|
|
git('checkout %s' % branch) |
|
|
|
@ -952,8 +954,8 @@ if need_to_regenerate_projects: |
|
|
|
if jobset.platform_string() == 'linux': |
|
|
|
if jobset.platform_string() == 'linux': |
|
|
|
subprocess.check_call('tools/buildgen/generate_projects.sh', shell=True) |
|
|
|
subprocess.check_call('tools/buildgen/generate_projects.sh', shell=True) |
|
|
|
else: |
|
|
|
else: |
|
|
|
print 'WARNING: may need to regenerate projects, but since we are not on' |
|
|
|
print('WARNING: may need to regenerate projects, but since we are not on') |
|
|
|
print ' Linux this step is being skipped. Compilation MAY fail.' |
|
|
|
print(' Linux this step is being skipped. Compilation MAY fail.') |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# grab config |
|
|
|
# grab config |
|
|
|
@ -980,18 +982,18 @@ for l in languages: |
|
|
|
language_make_options=[] |
|
|
|
language_make_options=[] |
|
|
|
if any(language.make_options() for language in languages): |
|
|
|
if any(language.make_options() for language in languages): |
|
|
|
if not 'gcov' in args.config and len(languages) != 1: |
|
|
|
if not 'gcov' in args.config and len(languages) != 1: |
|
|
|
print 'languages with custom make options cannot be built simultaneously with other languages' |
|
|
|
print('languages with custom make options cannot be built simultaneously with other languages') |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
else: |
|
|
|
else: |
|
|
|
language_make_options = next(iter(languages)).make_options() |
|
|
|
language_make_options = next(iter(languages)).make_options() |
|
|
|
|
|
|
|
|
|
|
|
if args.use_docker: |
|
|
|
if args.use_docker: |
|
|
|
if not args.travis: |
|
|
|
if not args.travis: |
|
|
|
print 'Seen --use_docker flag, will run tests under docker.' |
|
|
|
print('Seen --use_docker flag, will run tests under docker.') |
|
|
|
print |
|
|
|
print('') |
|
|
|
print 'IMPORTANT: The changes you are testing need to be locally committed' |
|
|
|
print('IMPORTANT: The changes you are testing need to be locally committed') |
|
|
|
print 'because only the committed changes in the current branch will be' |
|
|
|
print('because only the committed changes in the current branch will be') |
|
|
|
print 'copied to the docker environment.' |
|
|
|
print('copied to the docker environment.') |
|
|
|
time.sleep(5) |
|
|
|
time.sleep(5) |
|
|
|
|
|
|
|
|
|
|
|
dockerfile_dirs = set([l.dockerfile_dir() for l in languages]) |
|
|
|
dockerfile_dirs = set([l.dockerfile_dir() for l in languages]) |
|
|
|
@ -1075,7 +1077,7 @@ build_steps = list(set( |
|
|
|
for l in languages |
|
|
|
for l in languages |
|
|
|
for cmdline in l.pre_build_steps())) |
|
|
|
for cmdline in l.pre_build_steps())) |
|
|
|
if make_targets: |
|
|
|
if make_targets: |
|
|
|
make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems()) |
|
|
|
make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.items()) |
|
|
|
build_steps.extend(set(make_commands)) |
|
|
|
build_steps.extend(set(make_commands)) |
|
|
|
build_steps.extend(set( |
|
|
|
build_steps.extend(set( |
|
|
|
jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None) |
|
|
|
jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None) |
|
|
|
@ -1092,13 +1094,13 @@ forever = args.forever |
|
|
|
|
|
|
|
|
|
|
|
def _shut_down_legacy_server(legacy_server_port): |
|
|
|
def _shut_down_legacy_server(legacy_server_port): |
|
|
|
try: |
|
|
|
try: |
|
|
|
version = int(urllib2.urlopen( |
|
|
|
version = int(urllib.request.urlopen( |
|
|
|
'http://localhost:%d/version_number' % legacy_server_port, |
|
|
|
'http://localhost:%d/version_number' % legacy_server_port, |
|
|
|
timeout=10).read()) |
|
|
|
timeout=10).read()) |
|
|
|
except: |
|
|
|
except: |
|
|
|
pass |
|
|
|
pass |
|
|
|
else: |
|
|
|
else: |
|
|
|
urllib2.urlopen( |
|
|
|
urllib.request.urlopen( |
|
|
|
'http://localhost:%d/quitquitquit' % legacy_server_port).read() |
|
|
|
'http://localhost:%d/quitquitquit' % legacy_server_port).read() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -1120,29 +1122,29 @@ def _start_port_server(port_server_port): |
|
|
|
# if not running ==> start a new one |
|
|
|
# if not running ==> start a new one |
|
|
|
# otherwise, leave it up |
|
|
|
# otherwise, leave it up |
|
|
|
try: |
|
|
|
try: |
|
|
|
version = int(urllib2.urlopen( |
|
|
|
version = int(urllib.request.urlopen( |
|
|
|
'http://localhost:%d/version_number' % port_server_port, |
|
|
|
'http://localhost:%d/version_number' % port_server_port, |
|
|
|
timeout=10).read()) |
|
|
|
timeout=10).read()) |
|
|
|
print 'detected port server running version %d' % version |
|
|
|
print('detected port server running version %d' % version) |
|
|
|
running = True |
|
|
|
running = True |
|
|
|
except Exception as e: |
|
|
|
except Exception as e: |
|
|
|
print 'failed to detect port server: %s' % sys.exc_info()[0] |
|
|
|
print('failed to detect port server: %s' % sys.exc_info()[0]) |
|
|
|
print e.strerror |
|
|
|
print(e.strerror) |
|
|
|
running = False |
|
|
|
running = False |
|
|
|
if running: |
|
|
|
if running: |
|
|
|
current_version = int(subprocess.check_output( |
|
|
|
current_version = int(subprocess.check_output( |
|
|
|
[sys.executable, os.path.abspath('tools/run_tests/port_server.py'), |
|
|
|
[sys.executable, os.path.abspath('tools/run_tests/port_server.py'), |
|
|
|
'dump_version'])) |
|
|
|
'dump_version'])) |
|
|
|
print 'my port server is version %d' % current_version |
|
|
|
print('my port server is version %d' % current_version) |
|
|
|
running = (version >= current_version) |
|
|
|
running = (version >= current_version) |
|
|
|
if not running: |
|
|
|
if not running: |
|
|
|
print 'port_server version mismatch: killing the old one' |
|
|
|
print('port_server version mismatch: killing the old one') |
|
|
|
urllib2.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read() |
|
|
|
urllib.request.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read() |
|
|
|
time.sleep(1) |
|
|
|
time.sleep(1) |
|
|
|
if not running: |
|
|
|
if not running: |
|
|
|
fd, logfile = tempfile.mkstemp() |
|
|
|
fd, logfile = tempfile.mkstemp() |
|
|
|
os.close(fd) |
|
|
|
os.close(fd) |
|
|
|
print 'starting port_server, with log file %s' % logfile |
|
|
|
print('starting port_server, with log file %s' % logfile) |
|
|
|
args = [sys.executable, os.path.abspath('tools/run_tests/port_server.py'), |
|
|
|
args = [sys.executable, os.path.abspath('tools/run_tests/port_server.py'), |
|
|
|
'-p', '%d' % port_server_port, '-l', logfile] |
|
|
|
'-p', '%d' % port_server_port, '-l', logfile] |
|
|
|
env = dict(os.environ) |
|
|
|
env = dict(os.environ) |
|
|
|
@ -1168,34 +1170,34 @@ def _start_port_server(port_server_port): |
|
|
|
waits = 0 |
|
|
|
waits = 0 |
|
|
|
while True: |
|
|
|
while True: |
|
|
|
if waits > 10: |
|
|
|
if waits > 10: |
|
|
|
print 'killing port server due to excessive start up waits' |
|
|
|
print('killing port server due to excessive start up waits') |
|
|
|
port_server.kill() |
|
|
|
port_server.kill() |
|
|
|
if port_server.poll() is not None: |
|
|
|
if port_server.poll() is not None: |
|
|
|
print 'port_server failed to start' |
|
|
|
print('port_server failed to start') |
|
|
|
# try one final time: maybe another build managed to start one |
|
|
|
# try one final time: maybe another build managed to start one |
|
|
|
time.sleep(1) |
|
|
|
time.sleep(1) |
|
|
|
try: |
|
|
|
try: |
|
|
|
urllib2.urlopen('http://localhost:%d/get' % port_server_port, |
|
|
|
urllib.request.urlopen('http://localhost:%d/get' % port_server_port, |
|
|
|
timeout=1).read() |
|
|
|
timeout=1).read() |
|
|
|
print 'last ditch attempt to contact port server succeeded' |
|
|
|
print('last ditch attempt to contact port server succeeded') |
|
|
|
break |
|
|
|
break |
|
|
|
except: |
|
|
|
except: |
|
|
|
traceback.print_exc() |
|
|
|
traceback.print_exc() |
|
|
|
port_log = open(logfile, 'r').read() |
|
|
|
port_log = open(logfile, 'r').read() |
|
|
|
print port_log |
|
|
|
print(port_log) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
try: |
|
|
|
try: |
|
|
|
urllib2.urlopen('http://localhost:%d/get' % port_server_port, |
|
|
|
urllib.request.urlopen('http://localhost:%d/get' % port_server_port, |
|
|
|
timeout=1).read() |
|
|
|
timeout=1).read() |
|
|
|
print 'port server is up and ready' |
|
|
|
print('port server is up and ready') |
|
|
|
break |
|
|
|
break |
|
|
|
except socket.timeout: |
|
|
|
except socket.timeout: |
|
|
|
print 'waiting for port_server: timeout' |
|
|
|
print('waiting for port_server: timeout') |
|
|
|
traceback.print_exc(); |
|
|
|
traceback.print_exc(); |
|
|
|
time.sleep(1) |
|
|
|
time.sleep(1) |
|
|
|
waits += 1 |
|
|
|
waits += 1 |
|
|
|
except urllib2.URLError: |
|
|
|
except urllib.error.URLError: |
|
|
|
print 'waiting for port_server: urlerror' |
|
|
|
print('waiting for port_server: urlerror') |
|
|
|
traceback.print_exc(); |
|
|
|
traceback.print_exc(); |
|
|
|
time.sleep(1) |
|
|
|
time.sleep(1) |
|
|
|
waits += 1 |
|
|
|
waits += 1 |
|
|
|
|
Nicolas Noble
9 years ago
committed by
GitHub