Merge pull request #4647 from ctiller/run_errors

Handle errors in run_tests.py a little better

tools/run_tests/run_tests.py

@@ -961,6 +961,15 @@ def _calculate_num_runs_failures(list_of_results):
   return num_runs, num_failures
 
 
+# _build_and_run results
+class BuildAndRunError(object):
+
+  BUILD = object()
+  TEST = object()
+  POST_TEST = object()
+
+
+# returns a list of things that failed (or an empty list on success)
 def _build_and_run(
     check_cancelled, newline_on_success, cache, xml_report=None, build_only=False):
   """Do one pass of building & running tests."""
@@ -969,10 +978,10 @@ def _build_and_run(
       build_steps, maxjobs=1, stop_on_failure=True,
       newline_on_success=newline_on_success, travis=args.travis)
   if num_failures:
-    return 1
+    return [BuildAndRunError.BUILD]
 
   if build_only:
-    return 0
+    return []
 
   # start antagonists
   antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py'])
@@ -1030,12 +1039,16 @@ def _build_and_run(
   number_failures, _ = jobset.run(
       post_tests_steps, maxjobs=1, stop_on_failure=True,
       newline_on_success=newline_on_success, travis=args.travis)
-  if num_test_failures or number_failures:
-    return 2
+
+  out = []
+  if number_failures:
+    out.append(BuildAndRunError.POST_TEST)
+  if num_test_failures:
+    out.append(BuildAndRunError.TEST)
 
   if cache: cache.save()
 
-  return 0
+  return out
 
 
 test_cache = TestCache(runs_per_test == 1)
@@ -1048,11 +1061,11 @@ if forever:
     initial_time = dw.most_recent_change()
     have_files_changed = lambda: dw.most_recent_change() != initial_time
     previous_success = success
-    success = _build_and_run(check_cancelled=have_files_changed,
-                             newline_on_success=False,
-                             cache=test_cache,
-                             build_only=args.build_only) == 0
-    if not previous_success and success:
+    errors = _build_and_run(check_cancelled=have_files_changed,
+                            newline_on_success=False,
+                            cache=test_cache,
+                            build_only=args.build_only) == 0
+    if not previous_success and not errors:
       jobset.message('SUCCESS',
                      'All tests are now passing properly',
                      do_newline=True)
@@ -1060,13 +1073,21 @@ if forever:
     while not have_files_changed():
       time.sleep(1)
 else:
-  result = _build_and_run(check_cancelled=lambda: False,
+  errors = _build_and_run(check_cancelled=lambda: False,
                           newline_on_success=args.newline_on_success,
                           cache=test_cache,
                           xml_report=args.xml_report,
                           build_only=args.build_only)
-  if result == 0:
+  if not errors:
     jobset.message('SUCCESS', 'All tests passed', do_newline=True)
   else:
     jobset.message('FAILED', 'Some tests failed', do_newline=True)
-  sys.exit(result)
+  exit_code = 0
+  if BuildAndRunError.BUILD in errors:
+    exit_code |= 1
+  if BuildAndRunError.TEST in errors and not args.travis:
+    exit_code |= 2
+  if BuildAndRunError.POST_TEST in errors:
+    exit_code |= 4
+  sys.exit(exit_code)
+