The C based gRPC (C++, Python, Ruby, Objective-C, PHP, C#) https://grpc.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

249 lines
10 KiB

#!/usr/bin/env python2.7
# Copyright 2016, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tool to get build statistics from Jenkins and upload to BigQuery."""
import argparse
import jenkinsapi
from jenkinsapi.custom_exceptions import JenkinsAPIException
from jenkinsapi.jenkins import Jenkins
import json
import os
import re
import sys
import urllib
gcp_utils_dir = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../gcp/utils'))
sys.path.append(gcp_utils_dir)
import big_query_utils
_PROJECT_ID = 'grpc-testing'
_HAS_MATRIX = True
_BUILDS = {'gRPC_interop_master': not _HAS_MATRIX,
'gRPC_master_linux': not _HAS_MATRIX,
'gRPC_master_macos': not _HAS_MATRIX,
'gRPC_master_windows': not _HAS_MATRIX,
'gRPC_performance_master': not _HAS_MATRIX,
'gRPC_portability_master_linux': not _HAS_MATRIX,
'gRPC_portability_master_windows': not _HAS_MATRIX,
'gRPC_master_asanitizer_c': not _HAS_MATRIX,
'gRPC_master_asanitizer_cpp': not _HAS_MATRIX,
'gRPC_master_msan_c': not _HAS_MATRIX,
'gRPC_master_tsanitizer_c': not _HAS_MATRIX,
'gRPC_master_tsan_cpp': not _HAS_MATRIX,
'gRPC_interop_pull_requests': not _HAS_MATRIX,
'gRPC_performance_pull_requests': not _HAS_MATRIX,
'gRPC_portability_pull_requests_linux': not _HAS_MATRIX,
'gRPC_portability_pr_win': not _HAS_MATRIX,
'gRPC_pull_requests_linux': not _HAS_MATRIX,
'gRPC_pull_requests_macos': not _HAS_MATRIX,
'gRPC_pr_win': not _HAS_MATRIX,
'gRPC_pull_requests_asan_c': not _HAS_MATRIX,
'gRPC_pull_requests_asan_cpp': not _HAS_MATRIX,
'gRPC_pull_requests_msan_c': not _HAS_MATRIX,
'gRPC_pull_requests_tsan_c': not _HAS_MATRIX,
'gRPC_pull_requests_tsan_cpp': not _HAS_MATRIX,
}
_URL_BASE = 'https://grpc-testing.appspot.com/job'
# This is a dynamic list where known and active issues should be added.
# Fixed ones should be removed.
# Also try not to add multiple messages from the same failure.
_KNOWN_ERRORS = [
'Failed to build workspace Tests with scheme AllTests',
'Build timed out',
'TIMEOUT: tools/run_tests/pre_build_node.sh',
'TIMEOUT: tools/run_tests/pre_build_ruby.sh',
'FATAL: Unable to produce a script file',
8 years ago
'FAILED: build_docker_c\+\+',
'cannot find package \"cloud.google.com/go/compute/metadata\"',
'LLVM ERROR: IO failure on output stream.',
'MSBUILD : error MSB1009: Project file does not exist.',
'fatal: git fetch_pack: expected ACK/NAK',
'Failed to fetch from http://github.com/grpc/grpc.git',
('hudson.remoting.RemotingSystemException: java.io.IOException: '
'Backing channel is disconnected.'),
8 years ago
'hudson.remoting.ChannelClosedException',
'Could not initialize class hudson.Util',
'Too many open files in system',
'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=epoll',
'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=legacy',
'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=poll',
('tests.bins/asan/h2_proxy_test streaming_error_response '
'GRPC_POLL_STRATEGY=legacy'),
]
_NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found. Configuration error?'
_UNKNOWN_ERROR = 'Unknown error'
_DATASET_ID = 'build_statistics'
def _scrape_for_known_errors(html):
error_list = []
known_error_count = 0
for known_error in _KNOWN_ERRORS:
errors = re.findall(known_error, html)
this_error_count = len(errors)
if this_error_count > 0:
known_error_count += this_error_count
error_list.append({'description': known_error,
'count': this_error_count})
print('====> %d failures due to %s' % (this_error_count, known_error))
return error_list, known_error_count
def _no_report_files_found(html):
return _NO_REPORT_FILES_FOUND_ERROR in html
def _get_last_processed_buildnumber(build_name):
query = 'SELECT max(build_number) FROM [%s:%s.%s];' % (
_PROJECT_ID, _DATASET_ID, build_name)
query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query)
page = bq.jobs().getQueryResults(
pageToken=None,
**query_job['jobReference']).execute(num_retries=3)
if page['rows'][0]['f'][0]['v']:
return int(page['rows'][0]['f'][0]['v'])
return 0
def _process_matrix(build, url_base):
matrix_list = []
for matrix in build.get_matrix_runs():
matrix_str = re.match('.*\\xc2\\xbb ((?:[^,]+,?)+) #.*',
matrix.name).groups()[0]
matrix_tuple = matrix_str.split(',')
json_url = '%s/config=%s,language=%s,platform=%s/testReport/api/json' % (
url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
console_url = '%s/config=%s,language=%s,platform=%s/consoleFull' % (
url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
matrix_dict = {'name': matrix_str,
'duration': matrix.get_duration().total_seconds()}
matrix_dict.update(_process_build(json_url, console_url))
matrix_list.append(matrix_dict)
return matrix_list
def _process_build(json_url, console_url):
build_result = {}
error_list = []
try:
html = urllib.urlopen(json_url).read()
test_result = json.loads(html)
print('====> Parsing result from %s' % json_url)
failure_count = test_result['failCount']
build_result['pass_count'] = test_result['passCount']
build_result['failure_count'] = failure_count
build_result['no_report_files_found'] = _no_report_files_found(html)
if failure_count > 0:
error_list, known_error_count = _scrape_for_known_errors(html)
unknown_error_count = failure_count - known_error_count
# This can happen if the same error occurs multiple times in one test.
if failure_count < known_error_count:
print('====> Some errors are duplicates.')
unknown_error_count = 0
error_list.append({'description': _UNKNOWN_ERROR,
'count': unknown_error_count})
except Exception as e:
print('====> Got exception for %s: %s.' % (json_url, str(e)))
print('====> Parsing errors from %s.' % console_url)
html = urllib.urlopen(console_url).read()
build_result['pass_count'] = 0
build_result['failure_count'] = 1
error_list, _ = _scrape_for_known_errors(html)
if error_list:
error_list.append({'description': _UNKNOWN_ERROR, 'count': 0})
else:
error_list.append({'description': _UNKNOWN_ERROR, 'count': 1})
if error_list:
build_result['error'] = error_list
return build_result
# parse command line
argp = argparse.ArgumentParser(description='Get build statistics.')
argp.add_argument('-u', '--username', default='jenkins')
argp.add_argument('-b', '--builds',
choices=['all'] + sorted(_BUILDS.keys()),
nargs='+',
default=['all'])
args = argp.parse_args()
J = Jenkins('https://grpc-testing.appspot.com', args.username, 'apiToken')
bq = big_query_utils.create_big_query()
for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
print('====> Build: %s' % build_name)
# Since get_last_completed_build() always fails due to malformatted string
# error, we use get_build_metadata() instead.
job = None
try:
job = J[build_name]
except Exception as e:
print('====> Failed to get build %s: %s.' % (build_name, str(e)))
continue
last_processed_build_number = _get_last_processed_buildnumber(build_name)
last_complete_build_number = job.get_last_completed_buildnumber()
# To avoid processing all builds for a project never looked at. In this case,
# only examine 10 latest builds.
starting_build_number = max(last_processed_build_number+1,
last_complete_build_number-9)
for build_number in xrange(starting_build_number,
last_complete_build_number+1):
print('====> Processing %s build %d.' % (build_name, build_number))
build = None
try:
build = job.get_build_metadata(build_number)
except KeyError:
print('====> Build %s is missing. Skip.' % build_number)
continue
build_result = {'build_number': build_number,
'timestamp': str(build.get_timestamp())}
url_base = json_url = '%s/%s/%d' % (_URL_BASE, build_name, build_number)
if _BUILDS[build_name]: # The build has matrix, such as gRPC_master.
build_result['matrix'] = _process_matrix(build, url_base)
else:
json_url = '%s/testReport/api/json' % url_base
console_url = '%s/consoleFull' % url_base
build_result['duration'] = build.get_duration().total_seconds()
build_result.update(_process_build(json_url, console_url))
rows = [big_query_utils.make_row(build_number, build_result)]
if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name,
rows):
print '====> Error uploading result to bigquery.'
sys.exit(1)