|
|
|
#!/usr/bin/env python2.7
|
|
|
|
# Copyright 2016, Google Inc.
|
|
|
|
# All rights reserved.
|
|
|
|
#
|
|
|
|
# Redistribution and use in source and binary forms, with or without
|
|
|
|
# modification, are permitted provided that the following conditions are
|
|
|
|
# met:
|
|
|
|
#
|
|
|
|
# * Redistributions of source code must retain the above copyright
|
|
|
|
# notice, this list of conditions and the following disclaimer.
|
|
|
|
# * Redistributions in binary form must reproduce the above
|
|
|
|
# copyright notice, this list of conditions and the following disclaimer
|
|
|
|
# in the documentation and/or other materials provided with the
|
|
|
|
# distribution.
|
|
|
|
# * Neither the name of Google Inc. nor the names of its
|
|
|
|
# contributors may be used to endorse or promote products derived from
|
|
|
|
# this software without specific prior written permission.
|
|
|
|
#
|
|
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
|
|
"""Tool to get build statistics from Jenkins and upload to BigQuery."""
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
import jenkinsapi
|
|
|
|
from jenkinsapi.custom_exceptions import JenkinsAPIException
|
|
|
|
from jenkinsapi.jenkins import Jenkins
|
|
|
|
import json
|
|
|
|
import os
|
|
|
|
import re
|
|
|
|
import sys
|
|
|
|
import urllib
|
|
|
|
|
|
|
|
|
|
|
|
gcp_utils_dir = os.path.abspath(os.path.join(
|
|
|
|
os.path.dirname(__file__), '../gcp/utils'))
|
|
|
|
sys.path.append(gcp_utils_dir)
|
|
|
|
import big_query_utils
|
|
|
|
|
|
|
|
|
|
|
|
_PROJECT_ID = 'grpc-testing'
|
|
|
|
_HAS_MATRIX = True
|
|
|
|
_BUILDS = {'gRPC_interop_master': not _HAS_MATRIX,
|
|
|
|
'gRPC_master_linux': not _HAS_MATRIX,
|
|
|
|
'gRPC_master_macos': not _HAS_MATRIX,
|
|
|
|
'gRPC_master_windows': not _HAS_MATRIX,
|
|
|
|
'gRPC_performance_master': not _HAS_MATRIX,
|
|
|
|
'gRPC_portability_master_linux': not _HAS_MATRIX,
|
|
|
|
'gRPC_portability_master_windows': not _HAS_MATRIX,
|
|
|
|
'gRPC_master_asanitizer_c': not _HAS_MATRIX,
|
|
|
|
'gRPC_master_asanitizer_cpp': not _HAS_MATRIX,
|
|
|
|
'gRPC_master_msan_c': not _HAS_MATRIX,
|
|
|
|
'gRPC_master_tsanitizer_c': not _HAS_MATRIX,
|
|
|
|
'gRPC_master_tsan_cpp': not _HAS_MATRIX,
|
|
|
|
'gRPC_interop_pull_requests': not _HAS_MATRIX,
|
|
|
|
'gRPC_performance_pull_requests': not _HAS_MATRIX,
|
|
|
|
'gRPC_portability_pull_requests_linux': not _HAS_MATRIX,
|
|
|
|
'gRPC_portability_pr_win': not _HAS_MATRIX,
|
|
|
|
'gRPC_pull_requests_linux': not _HAS_MATRIX,
|
|
|
|
'gRPC_pull_requests_macos': not _HAS_MATRIX,
|
|
|
|
'gRPC_pr_win': not _HAS_MATRIX,
|
|
|
|
'gRPC_pull_requests_asan_c': not _HAS_MATRIX,
|
|
|
|
'gRPC_pull_requests_asan_cpp': not _HAS_MATRIX,
|
|
|
|
'gRPC_pull_requests_msan_c': not _HAS_MATRIX,
|
|
|
|
'gRPC_pull_requests_tsan_c': not _HAS_MATRIX,
|
|
|
|
'gRPC_pull_requests_tsan_cpp': not _HAS_MATRIX,
|
|
|
|
}
|
|
|
|
_URL_BASE = 'https://grpc-testing.appspot.com/job'
|
|
|
|
|
|
|
|
# This is a dynamic list where known and active issues should be added.
|
|
|
|
# Fixed ones should be removed.
|
|
|
|
# Also try not to add multiple messages from the same failure.
|
|
|
|
_KNOWN_ERRORS = [
|
|
|
|
'Failed to build workspace Tests with scheme AllTests',
|
|
|
|
'Build timed out',
|
|
|
|
'TIMEOUT: tools/run_tests/pre_build_node.sh',
|
|
|
|
'TIMEOUT: tools/run_tests/pre_build_ruby.sh',
|
|
|
|
'FATAL: Unable to produce a script file',
|
|
|
|
'FAILED: build_docker_c\+\+',
|
|
|
|
'cannot find package \"cloud.google.com/go/compute/metadata\"',
|
|
|
|
'LLVM ERROR: IO failure on output stream.',
|
|
|
|
'MSBUILD : error MSB1009: Project file does not exist.',
|
|
|
|
'fatal: git fetch_pack: expected ACK/NAK',
|
|
|
|
'Failed to fetch from http://github.com/grpc/grpc.git',
|
|
|
|
('hudson.remoting.RemotingSystemException: java.io.IOException: '
|
|
|
|
'Backing channel is disconnected.'),
|
|
|
|
'hudson.remoting.ChannelClosedException',
|
|
|
|
'Could not initialize class hudson.Util',
|
|
|
|
'Too many open files in system',
|
|
|
|
'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=epoll',
|
|
|
|
'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=legacy',
|
|
|
|
'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=poll',
|
|
|
|
('tests.bins/asan/h2_proxy_test streaming_error_response '
|
|
|
|
'GRPC_POLL_STRATEGY=legacy'),
|
|
|
|
]
|
|
|
|
_NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found. Configuration error?'
|
|
|
|
_UNKNOWN_ERROR = 'Unknown error'
|
|
|
|
_DATASET_ID = 'build_statistics'
|
|
|
|
|
|
|
|
|
|
|
|
def _scrape_for_known_errors(html):
|
|
|
|
error_list = []
|
|
|
|
known_error_count = 0
|
|
|
|
for known_error in _KNOWN_ERRORS:
|
|
|
|
errors = re.findall(known_error, html)
|
|
|
|
this_error_count = len(errors)
|
|
|
|
if this_error_count > 0:
|
|
|
|
known_error_count += this_error_count
|
|
|
|
error_list.append({'description': known_error,
|
|
|
|
'count': this_error_count})
|
|
|
|
print('====> %d failures due to %s' % (this_error_count, known_error))
|
|
|
|
return error_list, known_error_count
|
|
|
|
|
|
|
|
|
|
|
|
def _no_report_files_found(html):
|
|
|
|
return _NO_REPORT_FILES_FOUND_ERROR in html
|
|
|
|
|
|
|
|
|
|
|
|
def _get_last_processed_buildnumber(build_name):
|
|
|
|
query = 'SELECT max(build_number) FROM [%s:%s.%s];' % (
|
|
|
|
_PROJECT_ID, _DATASET_ID, build_name)
|
|
|
|
query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query)
|
|
|
|
page = bq.jobs().getQueryResults(
|
|
|
|
pageToken=None,
|
|
|
|
**query_job['jobReference']).execute(num_retries=3)
|
|
|
|
if page['rows'][0]['f'][0]['v']:
|
|
|
|
return int(page['rows'][0]['f'][0]['v'])
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
def _process_matrix(build, url_base):
|
|
|
|
matrix_list = []
|
|
|
|
for matrix in build.get_matrix_runs():
|
|
|
|
matrix_str = re.match('.*\\xc2\\xbb ((?:[^,]+,?)+) #.*',
|
|
|
|
matrix.name).groups()[0]
|
|
|
|
matrix_tuple = matrix_str.split(',')
|
|
|
|
json_url = '%s/config=%s,language=%s,platform=%s/testReport/api/json' % (
|
|
|
|
url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
|
|
|
|
console_url = '%s/config=%s,language=%s,platform=%s/consoleFull' % (
|
|
|
|
url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
|
|
|
|
matrix_dict = {'name': matrix_str,
|
|
|
|
'duration': matrix.get_duration().total_seconds()}
|
|
|
|
matrix_dict.update(_process_build(json_url, console_url))
|
|
|
|
matrix_list.append(matrix_dict)
|
|
|
|
|
|
|
|
return matrix_list
|
|
|
|
|
|
|
|
|
|
|
|
def _process_build(json_url, console_url):
|
|
|
|
build_result = {}
|
|
|
|
error_list = []
|
|
|
|
try:
|
|
|
|
html = urllib.urlopen(json_url).read()
|
|
|
|
test_result = json.loads(html)
|
|
|
|
print('====> Parsing result from %s' % json_url)
|
|
|
|
failure_count = test_result['failCount']
|
|
|
|
build_result['pass_count'] = test_result['passCount']
|
|
|
|
build_result['failure_count'] = failure_count
|
|
|
|
build_result['no_report_files_found'] = _no_report_files_found(html)
|
|
|
|
if failure_count > 0:
|
|
|
|
error_list, known_error_count = _scrape_for_known_errors(html)
|
|
|
|
unknown_error_count = failure_count - known_error_count
|
|
|
|
# This can happen if the same error occurs multiple times in one test.
|
|
|
|
if failure_count < known_error_count:
|
|
|
|
print('====> Some errors are duplicates.')
|
|
|
|
unknown_error_count = 0
|
|
|
|
error_list.append({'description': _UNKNOWN_ERROR,
|
|
|
|
'count': unknown_error_count})
|
|
|
|
except Exception as e:
|
|
|
|
print('====> Got exception for %s: %s.' % (json_url, str(e)))
|
|
|
|
print('====> Parsing errors from %s.' % console_url)
|
|
|
|
html = urllib.urlopen(console_url).read()
|
|
|
|
build_result['pass_count'] = 0
|
|
|
|
build_result['failure_count'] = 1
|
|
|
|
error_list, _ = _scrape_for_known_errors(html)
|
|
|
|
if error_list:
|
|
|
|
error_list.append({'description': _UNKNOWN_ERROR, 'count': 0})
|
|
|
|
else:
|
|
|
|
error_list.append({'description': _UNKNOWN_ERROR, 'count': 1})
|
|
|
|
|
|
|
|
if error_list:
|
|
|
|
build_result['error'] = error_list
|
|
|
|
|
|
|
|
return build_result
|
|
|
|
|
|
|
|
|
|
|
|
# parse command line
|
|
|
|
argp = argparse.ArgumentParser(description='Get build statistics.')
|
|
|
|
argp.add_argument('-u', '--username', default='jenkins')
|
|
|
|
argp.add_argument('-b', '--builds',
|
|
|
|
choices=['all'] + sorted(_BUILDS.keys()),
|
|
|
|
nargs='+',
|
|
|
|
default=['all'])
|
|
|
|
args = argp.parse_args()
|
|
|
|
|
|
|
|
J = Jenkins('https://grpc-testing.appspot.com', args.username, 'apiToken')
|
|
|
|
bq = big_query_utils.create_big_query()
|
|
|
|
|
|
|
|
for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
|
|
|
|
print('====> Build: %s' % build_name)
|
|
|
|
# Since get_last_completed_build() always fails due to malformatted string
|
|
|
|
# error, we use get_build_metadata() instead.
|
|
|
|
job = None
|
|
|
|
try:
|
|
|
|
job = J[build_name]
|
|
|
|
except Exception as e:
|
|
|
|
print('====> Failed to get build %s: %s.' % (build_name, str(e)))
|
|
|
|
continue
|
|
|
|
last_processed_build_number = _get_last_processed_buildnumber(build_name)
|
|
|
|
last_complete_build_number = job.get_last_completed_buildnumber()
|
|
|
|
# To avoid processing all builds for a project never looked at. In this case,
|
|
|
|
# only examine 10 latest builds.
|
|
|
|
starting_build_number = max(last_processed_build_number+1,
|
|
|
|
last_complete_build_number-9)
|
|
|
|
for build_number in xrange(starting_build_number,
|
|
|
|
last_complete_build_number+1):
|
|
|
|
print('====> Processing %s build %d.' % (build_name, build_number))
|
|
|
|
build = None
|
|
|
|
try:
|
|
|
|
build = job.get_build_metadata(build_number)
|
|
|
|
except KeyError:
|
|
|
|
print('====> Build %s is missing. Skip.' % build_number)
|
|
|
|
continue
|
|
|
|
build_result = {'build_number': build_number,
|
|
|
|
'timestamp': str(build.get_timestamp())}
|
|
|
|
url_base = json_url = '%s/%s/%d' % (_URL_BASE, build_name, build_number)
|
|
|
|
if _BUILDS[build_name]: # The build has matrix, such as gRPC_master.
|
|
|
|
build_result['matrix'] = _process_matrix(build, url_base)
|
|
|
|
else:
|
|
|
|
json_url = '%s/testReport/api/json' % url_base
|
|
|
|
console_url = '%s/consoleFull' % url_base
|
|
|
|
build_result['duration'] = build.get_duration().total_seconds()
|
|
|
|
build_result.update(_process_build(json_url, console_url))
|
|
|
|
rows = [big_query_utils.make_row(build_number, build_result)]
|
|
|
|
if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name,
|
|
|
|
rows):
|
|
|
|
print '====> Error uploading result to bigquery.'
|
|
|
|
sys.exit(1)
|
|
|
|
|