Merge pull request #12136 from adelez/build_stats3

Correct no_report_files_found and error fields

tools/run_tests/run_build_statistics.py

@@ -88,24 +88,26 @@ _KNOWN_ERRORS = [
     'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=poll',
     ('tests.bins/asan/h2_proxy_test streaming_error_response '
      'GRPC_POLL_STRATEGY=legacy'),
+    'hudson.plugins.git.GitException',
+    'Couldn\'t find any revision to build',
+    'org.jenkinsci.plugin.Diskcheck.preCheckout',
+    'Something went wrong while deleting Files',
 ]
-_NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found. Configuration error?'
+_NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found.'
 _UNKNOWN_ERROR = 'Unknown error'
 _DATASET_ID = 'build_statistics'
 
 
 def _scrape_for_known_errors(html):
   error_list = []
-  known_error_count = 0
   for known_error in _KNOWN_ERRORS:
     errors = re.findall(known_error, html)
     this_error_count = len(errors)
     if this_error_count > 0: 
-      known_error_count += this_error_count
       error_list.append({'description': known_error,
                          'count': this_error_count})
       print('====> %d failures due to %s' % (this_error_count, known_error))
-  return error_list, known_error_count
+  return error_list
 
 
 def _no_report_files_found(html):
@@ -156,23 +158,23 @@ def _process_build(json_url, console_url):
     build_result['no_report_files_found'] = _no_report_files_found(html)
     # Only check errors if Jenkins failure occurred.
     if build_result['no_report_files_found']:
-      error_list, known_error_count = _scrape_for_known_errors(html)
-      if not error_list:
-        error_list.append({'description': _UNKNOWN_ERROR, 'count': 1})
+      error_list = _scrape_for_known_errors(html)
   except Exception as e:
     print('====> Got exception for %s: %s.' % (json_url, str(e)))   
     print('====> Parsing errors from %s.' % console_url)
     html = urllib.urlopen(console_url).read()
     build_result['pass_count'] = 0  
     build_result['failure_count'] = 1
-    error_list, _ = _scrape_for_known_errors(html)
-    if error_list:
-      error_list.append({'description': _UNKNOWN_ERROR, 'count': 0})
-    else:
-      error_list.append({'description': _UNKNOWN_ERROR, 'count': 1})
- 
+    # In this case, the string doesn't exist in the result html but the fact 
+    # that we fail to parse the result html indicates Jenkins failure and hence 
+    # no report files were generated.
+    build_result['no_report_files_found'] = True
+    error_list = _scrape_for_known_errors(html)
+
   if error_list:
     build_result['error'] = error_list
+  elif build_result['no_report_files_found']:
+    build_result['error'] = [{'description': _UNKNOWN_ERROR, 'count': 1}]
   else:
     build_result['error'] = [{'description': '', 'count': 0}]
 
@@ -213,6 +215,15 @@ for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
     build = None
     try:
       build = job.get_build_metadata(build_number)
+      print('====> Build status: %s.' % build.get_status())
+      if build.get_status() == 'ABORTED':
+        continue
+      # If any build is still running, stop processing this job. Next time, we
+      # start from where it was left so that all builds are processed 
+      # sequentially.
+      if build.is_running():
+        print('====> Build %d is still running.' % build_number)
+        break
     except KeyError:
       print('====> Build %s is missing. Skip.' % build_number)
       continue
@@ -225,10 +236,10 @@ for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
       json_url = '%s/testReport/api/json' % url_base
       console_url = '%s/consoleFull' % url_base
       build_result['duration'] = build.get_duration().total_seconds()
-      build_result.update(_process_build(json_url, console_url))
+      build_stat = _process_build(json_url, console_url)
+      build_result.update(build_stat)
     rows = [big_query_utils.make_row(build_number, build_result)]
     if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name, 
                                        rows):
       print('====> Error uploading result to bigquery.')
       sys.exit(1)
-