Merge pull request #3487 from ctiller/flaky-timeouts

Add support for flaky timeouts
pull/3554/head
Jan Tattermusch 10 years ago
commit 10ce383e1b
  1. 27
      tools/run_tests/jobset.py
  2. 3
      tools/run_tests/run_tests.py

@ -83,6 +83,7 @@ _CLEAR_LINE = '\x1b[2K'
_TAG_COLOR = {
'FAILED': 'red',
'FLAKE': 'purple',
'TIMEOUT_FLAKE': 'purple',
'WARNING': 'yellow',
'TIMEOUT': 'red',
'PASSED': 'green',
@ -133,7 +134,8 @@ class JobSpec(object):
"""Specifies what to run for a job."""
def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None,
cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0):
cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0,
timeout_retries=0):
"""
Arguments:
cmdline: a list of arguments to pass as the command line
@ -153,6 +155,7 @@ class JobSpec(object):
self.shell = shell
self.timeout_seconds = timeout_seconds
self.flake_retries = flake_retries
self.timeout_retries = timeout_retries
def identity(self):
return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets)
@ -176,6 +179,7 @@ class Job(object):
self._xml_test = ET.SubElement(xml_report, 'testcase',
name=self._spec.shortname) if xml_report is not None else None
self._retries = 0
self._timeout_retries = 0
message('START', spec.shortname, do_newline=self._travis)
self.start()
@ -223,19 +227,26 @@ class Job(object):
ET.SubElement(self._xml_test, 'failure', message='Failure').text
else:
self._state = _SUCCESS
message('PASSED', '%s [time=%.1fsec; retries=%d]' % (self._spec.shortname, elapsed, self._retries),
do_newline=self._newline_on_success or self._travis)
message('PASSED', '%s [time=%.1fsec; retries=%d;%d]' % (
self._spec.shortname, elapsed, self._retries, self._timeout_retries),
do_newline=self._newline_on_success or self._travis)
if self._bin_hash:
update_cache.finished(self._spec.identity(), self._bin_hash)
elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds:
self._tempfile.seek(0)
stdout = self._tempfile.read()
filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore'))
message('TIMEOUT', self._spec.shortname, stdout, do_newline=True)
self.kill()
if self._xml_test is not None:
ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
ET.SubElement(self._xml_test, 'error', message='Timeout')
if self._timeout_retries < self._spec.timeout_retries:
message('TIMEOUT_FLAKE', self._spec.shortname, stdout, do_newline=True)
self._timeout_retries += 1
self._process.terminate()
self.start()
else:
message('TIMEOUT', self._spec.shortname, stdout, do_newline=True)
self.kill()
if self._xml_test is not None:
ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout
ET.SubElement(self._xml_test, 'error', message='Timeout')
return self._state
def kill(self):

@ -120,7 +120,8 @@ class ValgrindConfig(object):
self.args + cmdline,
shortname='valgrind %s' % cmdline[0],
hash_targets=None,
flake_retries=5 if args.allow_flakes else 0)
flake_retries=5 if args.allow_flakes else 0,
timeout_retries=2 if args.allow_flakes else 0)
def get_c_tests(travis, test_lang) :

Loading…
Cancel
Save