grpc/tools/run_tests/run_build_statistics.py

#!/usr/bin/env python
# Copyright 2016 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tool to get build statistics from Jenkins and upload to BigQuery."""

from __future__ import print_function

import argparse
import jenkinsapi
from jenkinsapi.custom_exceptions import JenkinsAPIException
from jenkinsapi.jenkins import Jenkins
import json
import os
import re
import sys
import urllib

gcp_utils_dir = os.path.abspath(
    os.path.join(os.path.dirname(__file__), '../gcp/utils'))
sys.path.append(gcp_utils_dir)
import big_query_utils

_PROJECT_ID = 'grpc-testing'
_HAS_MATRIX = True
_BUILDS = {
    'gRPC_interop_master': not _HAS_MATRIX,
    'gRPC_master_linux': not _HAS_MATRIX,
    'gRPC_master_macos': not _HAS_MATRIX,
    'gRPC_master_windows': not _HAS_MATRIX,
    'gRPC_performance_master': not _HAS_MATRIX,
    'gRPC_portability_master_linux': not _HAS_MATRIX,
    'gRPC_portability_master_windows': not _HAS_MATRIX,
    'gRPC_master_asanitizer_c': not _HAS_MATRIX,
    'gRPC_master_asanitizer_cpp': not _HAS_MATRIX,
    'gRPC_master_msan_c': not _HAS_MATRIX,
    'gRPC_master_tsanitizer_c': not _HAS_MATRIX,
    'gRPC_master_tsan_cpp': not _HAS_MATRIX,
    'gRPC_interop_pull_requests': not _HAS_MATRIX,
    'gRPC_performance_pull_requests': not _HAS_MATRIX,
    'gRPC_portability_pull_requests_linux': not _HAS_MATRIX,
    'gRPC_portability_pr_win': not _HAS_MATRIX,
    'gRPC_pull_requests_linux': not _HAS_MATRIX,
    'gRPC_pull_requests_macos': not _HAS_MATRIX,
    'gRPC_pr_win': not _HAS_MATRIX,
    'gRPC_pull_requests_asan_c': not _HAS_MATRIX,
    'gRPC_pull_requests_asan_cpp': not _HAS_MATRIX,
    'gRPC_pull_requests_msan_c': not _HAS_MATRIX,
    'gRPC_pull_requests_tsan_c': not _HAS_MATRIX,
    'gRPC_pull_requests_tsan_cpp': not _HAS_MATRIX,
}
_URL_BASE = 'https://grpc-testing.appspot.com/job'

# This is a dynamic list where known and active issues should be added. 
# Fixed ones should be removed.
# Also try not to add multiple messages from the same failure.
_KNOWN_ERRORS = [
    'Failed to build workspace Tests with scheme AllTests',
    'Build timed out',
    'TIMEOUT: tools/run_tests/pre_build_node.sh',
    'TIMEOUT: tools/run_tests/pre_build_ruby.sh',
    'FATAL: Unable to produce a script file',
    'FAILED: build_docker_c\+\+',
    'cannot find package \"cloud.google.com/go/compute/metadata\"',
    'LLVM ERROR: IO failure on output stream.',
    'MSBUILD : error MSB1009: Project file does not exist.',
    'fatal: git fetch_pack: expected ACK/NAK',
    'Failed to fetch from http://github.com/grpc/grpc.git',
    ('hudson.remoting.RemotingSystemException: java.io.IOException: '
     'Backing channel is disconnected.'),
    'hudson.remoting.ChannelClosedException',
    'Could not initialize class hudson.Util',
    'Too many open files in system',
    'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=epoll',
    'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=legacy',
    'FAILED: bins/tsan/qps_openloop_test GRPC_POLL_STRATEGY=poll',
    ('tests.bins/asan/h2_proxy_test streaming_error_response '
     'GRPC_POLL_STRATEGY=legacy'),
    'hudson.plugins.git.GitException',
    'Couldn\'t find any revision to build',
    'org.jenkinsci.plugin.Diskcheck.preCheckout',
    'Something went wrong while deleting Files',
]
_NO_REPORT_FILES_FOUND_ERROR = 'No test report files were found.'
_UNKNOWN_ERROR = 'Unknown error'
_DATASET_ID = 'build_statistics'


def _scrape_for_known_errors(html):
    error_list = []
    for known_error in _KNOWN_ERRORS:
        errors = re.findall(known_error, html)
        this_error_count = len(errors)
        if this_error_count > 0:
            error_list.append({
                'description': known_error,
                'count': this_error_count
            })
            print('====> %d failures due to %s' %
                  (this_error_count, known_error))
    return error_list


def _no_report_files_found(html):
    return _NO_REPORT_FILES_FOUND_ERROR in html


def _get_last_processed_buildnumber(build_name):
    query = 'SELECT max(build_number) FROM [%s:%s.%s];' % (
        _PROJECT_ID, _DATASET_ID, build_name)
    query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query)
    page = bq.jobs().getQueryResults(
        pageToken=None, **query_job['jobReference']).execute(num_retries=3)
    if page['rows'][0]['f'][0]['v']:
        return int(page['rows'][0]['f'][0]['v'])
    return 0


def _process_matrix(build, url_base):
    matrix_list = []
    for matrix in build.get_matrix_runs():
        matrix_str = re.match('.*\\xc2\\xbb ((?:[^,]+,?)+) #.*',
                              matrix.name).groups()[0]
        matrix_tuple = matrix_str.split(',')
        json_url = '%s/config=%s,language=%s,platform=%s/testReport/api/json' % (
            url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
        console_url = '%s/config=%s,language=%s,platform=%s/consoleFull' % (
            url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2])
        matrix_dict = {
            'name': matrix_str,
            'duration': matrix.get_duration().total_seconds()
        }
        matrix_dict.update(_process_build(json_url, console_url))
        matrix_list.append(matrix_dict)

    return matrix_list


def _process_build(json_url, console_url):
    build_result = {}
    error_list = []
    try:
        html = urllib.urlopen(json_url).read()
        test_result = json.loads(html)
        print('====> Parsing result from %s' % json_url)
        failure_count = test_result['failCount']
        build_result['pass_count'] = test_result['passCount']
        build_result['failure_count'] = failure_count
        # This means Jenkins failure occurred.
        build_result['no_report_files_found'] = _no_report_files_found(html)
        # Only check errors if Jenkins failure occurred.
        if build_result['no_report_files_found']:
            error_list = _scrape_for_known_errors(html)
    except Exception as e:
        print('====> Got exception for %s: %s.' % (json_url, str(e)))
        print('====> Parsing errors from %s.' % console_url)
        html = urllib.urlopen(console_url).read()
        build_result['pass_count'] = 0
        build_result['failure_count'] = 1
        # In this case, the string doesn't exist in the result html but the fact 
        # that we fail to parse the result html indicates Jenkins failure and hence 
        # no report files were generated.
        build_result['no_report_files_found'] = True
        error_list = _scrape_for_known_errors(html)

    if error_list:
        build_result['error'] = error_list
    elif build_result['no_report_files_found']:
        build_result['error'] = [{'description': _UNKNOWN_ERROR, 'count': 1}]
    else:
        build_result['error'] = [{'description': '', 'count': 0}]

    return build_result


# parse command line
argp = argparse.ArgumentParser(description='Get build statistics.')
argp.add_argument('-u', '--username', default='jenkins')
argp.add_argument(
    '-b',
    '--builds',
    choices=['all'] + sorted(_BUILDS.keys()),
    nargs='+',
    default=['all'])
args = argp.parse_args()

J = Jenkins('https://grpc-testing.appspot.com', args.username, 'apiToken')
bq = big_query_utils.create_big_query()

for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
    print('====> Build: %s' % build_name)
    # Since get_last_completed_build() always fails due to malformatted string
    # error, we use get_build_metadata() instead.
    job = None
    try:
        job = J[build_name]
    except Exception as e:
        print('====> Failed to get build %s: %s.' % (build_name, str(e)))
        continue
    last_processed_build_number = _get_last_processed_buildnumber(build_name)
    last_complete_build_number = job.get_last_completed_buildnumber()
    # To avoid processing all builds for a project never looked at. In this case,
    # only examine 10 latest builds.
    starting_build_number = max(last_processed_build_number + 1,
                                last_complete_build_number - 9)
    for build_number in xrange(starting_build_number,
                               last_complete_build_number + 1):
        print('====> Processing %s build %d.' % (build_name, build_number))
        build = None
        try:
            build = job.get_build_metadata(build_number)
            print('====> Build status: %s.' % build.get_status())
            if build.get_status() == 'ABORTED':
                continue
            # If any build is still running, stop processing this job. Next time, we
            # start from where it was left so that all builds are processed 
            # sequentially.
            if build.is_running():
                print('====> Build %d is still running.' % build_number)
                break
        except KeyError:
            print('====> Build %s is missing. Skip.' % build_number)
            continue
        build_result = {
            'build_number': build_number,
            'timestamp': str(build.get_timestamp())
        }
        url_base = json_url = '%s/%s/%d' % (_URL_BASE, build_name, build_number)
        if _BUILDS[build_name]:  # The build has matrix, such as gRPC_master.
            build_result['matrix'] = _process_matrix(build, url_base)
        else:
            json_url = '%s/testReport/api/json' % url_base
            console_url = '%s/consoleFull' % url_base
            build_result['duration'] = build.get_duration().total_seconds()
            build_stat = _process_build(json_url, console_url)
            build_result.update(build_stat)
        rows = [big_query_utils.make_row(build_number, build_result)]
        if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID,
                                           build_name, rows):
            print('====> Error uploading result to bigquery.')
            sys.exit(1)