mirror of https://github.com/grpc/grpc.git
The C based gRPC (C++, Python, Ruby, Objective-C, PHP, C#)
https://grpc.io/
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
250 lines
10 KiB
250 lines
10 KiB
#!/usr/bin/env python |
|
# Copyright 2016 gRPC authors. |
|
# |
|
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
# you may not use this file except in compliance with the License. |
|
# You may obtain a copy of the License at |
|
# |
|
# http://www.apache.org/licenses/LICENSE-2.0 |
|
# |
|
# Unless required by applicable law or agreed to in writing, software |
|
# distributed under the License is distributed on an "AS IS" BASIS, |
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
# See the License for the specific language governing permissions and |
|
# limitations under the License. |
|
"""Tool to get build statistics from Jenkins and upload to BigQuery.""" |
|
|
|
from __future__ import print_function |
|
|
|
import argparse |
|
import jenkinsapi |
|
from jenkinsapi.custom_exceptions import JenkinsAPIException |
|
from jenkinsapi.jenkins import Jenkins |
|
import json |
|
import os |
|
import re |
|
import sys |
|
import urllib |
|
|
|
gcp_utils_dir = os.path.abspath( |
|
os.path.join(os.path.dirname(__file__), '../gcp/utils')) |
|
sys.path.append(gcp_utils_dir) |
|
import big_query_utils |
|
|
|
_PROJECT_ID = 'grpc-testing' |
|
_HAS_MATRIX = True |
|
_BUILDS = { |
|
'gRPC_interop_master': not _HAS_MATRIX, |
|
'gRPC_master_linux': not _HAS_MATRIX, |
|
'gRPC_master_macos': not _HAS_MATRIX, |
|
'gRPC_master_windows': not _HAS_MATRIX, |
|
'gRPC_performance_master': not _HAS_MATRIX, |
|
'gRPC_portability_master_linux': not _HAS_MATRIX, |
|
'gRPC_portability_master_windows': not _HAS_MATRIX, |
|
'gRPC_master_asanitizer_c': not _HAS_MATRIX, |
|
'gRPC_master_asanitizer_cpp': not _HAS_MATRIX, |
|
'gRPC_master_msan_c': not _HAS_MATRIX, |
|
'gRPC_master_tsanitizer_c': not _HAS_MATRIX, |
|
'gRPC_master_tsan_cpp': not _HAS_MATRIX, |
|
'gRPC_interop_pull_requests': not _HAS_MATRIX, |
|
'gRPC_performance_pull_requests': not _HAS_MATRIX, |
|
'gRPC_portability_pull_requests_linux': not _HAS_MATRIX, |
|
'gRPC_portability_pr_win': not _HAS_MATRIX, |
|
'gRPC_pull_requests_linux': not _HAS_MATRIX, |
|
'gRPC_pull_requests_macos': not _HAS_MATRIX, |
|
'gRPC_pr_win': not _HAS_MATRIX, |
|
'gRPC_pull_requests_asan_c': not _HAS_MATRIX, |
|
'gRPC_pull_requests_asan_cpp': not _HAS_MATRIX, |
|
'gRPC_pull_requests_msan_c': not _HAS_MATRIX, |
|
'gRPC_pull_requests_tsan_c': not _HAS_MATRIX, |
|
'gRPC_pull_requests_tsan_cpp': not _HAS_MATRIX, |
|
} |
|
_URL_BASE = 'https://grpc-testing.appspot.com/job' |
|
|
|
# This is a dynamic list where known and active issues should be added. |
|
# Fixed ones should be removed. |
|
# Also try not to add multiple messages from the same failure. |
|
_KNOWN_ERRORS = [ |
|
'Failed to build workspace Tests with scheme AllTests', |
|
'Build timed out', |
|
'TIMEOUT: tools/run_tests/pre_build_node.sh', |
|
'TIMEOUT: tools/run_tests/pre_build_ruby.sh', |
|
'FATAL: Unable to produce a script file', |
|
'FAILED: build_docker_c\+\+', |
|
'cannot find package \"cloud.google.com/go/compute/metadata\"', |
|
'LLVM ERROR: IO failure on output stream.', |
|
'MSBUILD : error MSB1009: Project file does not exist.', |
|
'fatal: git fetch_pack: expected ACK/NAK', |
|
'Failed to fetch from http://github.com/grpc/grpc.git', |
|
('hudson.remoting.RemotingSystemException: java.io.IOException: ' |
|
'Backing channel is disconnected.'), |
|
'hudson.remoting.ChannelClosedException', |
|
'Could not initialize class hudson.Util', |
|
'Too many open files in system', |
|
'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=epoll', |
|
'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=legacy', |
|
'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=poll', |
|
('tests.bins/asan/h2_proxy_test streaming_error_response ' |
|
'GRPC_POLL_STRATEGY=legacy'), |
|
'hudson.plugins.git.GitException', |
|
'Couldn\'t find any revision to build', |
|
'org.jenkinsci.plugin.Diskcheck.preCheckout', |
|
'Something went wrong while deleting Files', |
|
] |
|
_NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found.' |
|
_UNKNOWN_ERROR = 'Unknown error' |
|
_DATASET_ID = 'build_statistics' |
|
|
|
|
|
def _scrape_for_known_errors(html): |
|
error_list = [] |
|
for known_error in _KNOWN_ERRORS: |
|
errors = re.findall(known_error, html) |
|
this_error_count = len(errors) |
|
if this_error_count > 0: |
|
error_list.append({ |
|
'description': known_error, |
|
'count': this_error_count |
|
}) |
|
print('====> %d failures due to %s' % |
|
(this_error_count, known_error)) |
|
return error_list |
|
|
|
|
|
def _no_report_files_found(html): |
|
return _NO_REPORT_FILES_FOUND_ERROR in html |
|
|
|
|
|
def _get_last_processed_buildnumber(build_name): |
|
query = 'SELECT max(build_number) FROM [%s:%s.%s];' % ( |
|
_PROJECT_ID, _DATASET_ID, build_name) |
|
query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query) |
|
page = bq.jobs().getQueryResults( |
|
pageToken=None, **query_job['jobReference']).execute(num_retries=3) |
|
if page['rows'][0]['f'][0]['v']: |
|
return int(page['rows'][0]['f'][0]['v']) |
|
return 0 |
|
|
|
|
|
def _process_matrix(build, url_base): |
|
matrix_list = [] |
|
for matrix in build.get_matrix_runs(): |
|
matrix_str = re.match('.*\\xc2\\xbb ((?:[^,]+,?)+) #.*', |
|
matrix.name).groups()[0] |
|
matrix_tuple = matrix_str.split(',') |
|
json_url = '%s/config=%s,language=%s,platform=%s/testReport/api/json' % ( |
|
url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2]) |
|
console_url = '%s/config=%s,language=%s,platform=%s/consoleFull' % ( |
|
url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2]) |
|
matrix_dict = { |
|
'name': matrix_str, |
|
'duration': matrix.get_duration().total_seconds() |
|
} |
|
matrix_dict.update(_process_build(json_url, console_url)) |
|
matrix_list.append(matrix_dict) |
|
|
|
return matrix_list |
|
|
|
|
|
def _process_build(json_url, console_url): |
|
build_result = {} |
|
error_list = [] |
|
try: |
|
html = urllib.urlopen(json_url).read() |
|
test_result = json.loads(html) |
|
print('====> Parsing result from %s' % json_url) |
|
failure_count = test_result['failCount'] |
|
build_result['pass_count'] = test_result['passCount'] |
|
build_result['failure_count'] = failure_count |
|
# This means Jenkins failure occurred. |
|
build_result['no_report_files_found'] = _no_report_files_found(html) |
|
# Only check errors if Jenkins failure occurred. |
|
if build_result['no_report_files_found']: |
|
error_list = _scrape_for_known_errors(html) |
|
except Exception as e: |
|
print('====> Got exception for %s: %s.' % (json_url, str(e))) |
|
print('====> Parsing errors from %s.' % console_url) |
|
html = urllib.urlopen(console_url).read() |
|
build_result['pass_count'] = 0 |
|
build_result['failure_count'] = 1 |
|
# In this case, the string doesn't exist in the result html but the fact |
|
# that we fail to parse the result html indicates Jenkins failure and hence |
|
# no report files were generated. |
|
build_result['no_report_files_found'] = True |
|
error_list = _scrape_for_known_errors(html) |
|
|
|
if error_list: |
|
build_result['error'] = error_list |
|
elif build_result['no_report_files_found']: |
|
build_result['error'] = [{'description': _UNKNOWN_ERROR, 'count': 1}] |
|
else: |
|
build_result['error'] = [{'description': '', 'count': 0}] |
|
|
|
return build_result |
|
|
|
|
|
# parse command line |
|
argp = argparse.ArgumentParser(description='Get build statistics.') |
|
argp.add_argument('-u', '--username', default='jenkins') |
|
argp.add_argument('-b', |
|
'--builds', |
|
choices=['all'] + sorted(_BUILDS.keys()), |
|
nargs='+', |
|
default=['all']) |
|
args = argp.parse_args() |
|
|
|
J = Jenkins('https://grpc-testing.appspot.com', args.username, 'apiToken') |
|
bq = big_query_utils.create_big_query() |
|
|
|
for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds: |
|
print('====> Build: %s' % build_name) |
|
# Since get_last_completed_build() always fails due to malformatted string |
|
# error, we use get_build_metadata() instead. |
|
job = None |
|
try: |
|
job = J[build_name] |
|
except Exception as e: |
|
print('====> Failed to get build %s: %s.' % (build_name, str(e))) |
|
continue |
|
last_processed_build_number = _get_last_processed_buildnumber(build_name) |
|
last_complete_build_number = job.get_last_completed_buildnumber() |
|
# To avoid processing all builds for a project never looked at. In this case, |
|
# only examine 10 latest builds. |
|
starting_build_number = max(last_processed_build_number + 1, |
|
last_complete_build_number - 9) |
|
for build_number in xrange(starting_build_number, |
|
last_complete_build_number + 1): |
|
print('====> Processing %s build %d.' % (build_name, build_number)) |
|
build = None |
|
try: |
|
build = job.get_build_metadata(build_number) |
|
print('====> Build status: %s.' % build.get_status()) |
|
if build.get_status() == 'ABORTED': |
|
continue |
|
# If any build is still running, stop processing this job. Next time, we |
|
# start from where it was left so that all builds are processed |
|
# sequentially. |
|
if build.is_running(): |
|
print('====> Build %d is still running.' % build_number) |
|
break |
|
except KeyError: |
|
print('====> Build %s is missing. Skip.' % build_number) |
|
continue |
|
build_result = { |
|
'build_number': build_number, |
|
'timestamp': str(build.get_timestamp()) |
|
} |
|
url_base = json_url = '%s/%s/%d' % (_URL_BASE, build_name, build_number) |
|
if _BUILDS[build_name]: # The build has matrix, such as gRPC_master. |
|
build_result['matrix'] = _process_matrix(build, url_base) |
|
else: |
|
json_url = '%s/testReport/api/json' % url_base |
|
console_url = '%s/consoleFull' % url_base |
|
build_result['duration'] = build.get_duration().total_seconds() |
|
build_stat = _process_build(json_url, console_url) |
|
build_result.update(build_stat) |
|
rows = [big_query_utils.make_row(build_number, build_result)] |
|
if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, |
|
build_name, rows): |
|
print('====> Error uploading result to bigquery.') |
|
sys.exit(1)
|
|
|