From dbd03b5e2d97f869089b2165edd2c419101b1c83 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 31 May 2016 13:00:33 -0700 Subject: [PATCH 1/6] Make client_channels be across all clients, not per-client --- test/cpp/qps/driver.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 04b2b453f9e..da47a985b8c 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -306,6 +306,7 @@ std::unique_ptr RunScenario( // clients is array rather than std::vector to avoid gcc-4.4 issues // where class contained in std::vector must have a copy constructor auto* clients = new ClientData[num_clients]; + size_t channels_allocated = 0; for (size_t i = 0; i < num_clients; i++) { const auto& worker = workers[i + num_servers]; gpr_log(GPR_INFO, "Starting client on %s (worker #%d)", worker.c_str(), @@ -341,6 +342,13 @@ std::unique_ptr RunScenario( } } + size_t num_channels = + (i == num_clients - 1) + ? channels_allocated - client_config.client_channels() + : client_config.client_channels() / num_clients; + channels_allocated += num_channels; + per_client_config.set_client_channels(num_channels); + ClientArgs args; *args.mutable_setup() = per_client_config; clients[i].stream = From 1adb13143f21a54a3e3d7018d0db4d15d78c1adc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 31 May 2016 13:04:32 -0700 Subject: [PATCH 2/6] Add comment --- test/cpp/qps/driver.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index da47a985b8c..147c540840e 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -342,6 +342,8 @@ std::unique_ptr RunScenario( } } + // Reduce channel count so that total channels specified is held regardless + // of the number of clients available size_t num_channels = (i == num_clients - 1) ? channels_allocated - client_config.client_channels() From d4aa7cf5a502462debd4a42f0daf4c018c824263 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 31 May 2016 13:59:25 -0700 Subject: [PATCH 3/6] Fix negation --- test/cpp/qps/driver.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 147c540840e..298087dd36f 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -346,9 +346,10 @@ std::unique_ptr RunScenario( // of the number of clients available size_t num_channels = (i == num_clients - 1) - ? channels_allocated - client_config.client_channels() + ? client_config.client_channels() - channels_allocated : client_config.client_channels() / num_clients; channels_allocated += num_channels; + gpr_log(GPR_DEBUG, "Client %d gets %d channels", i, num_channels); per_client_config.set_client_channels(num_channels); ClientArgs args; From 7e5d46496b6ad47aa84583260cf7886f7f51da5f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 22 Jun 2016 08:17:27 -0700 Subject: [PATCH 4/6] Better distribution --- test/cpp/qps/driver.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 147c540840e..2efdc1288ff 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -345,9 +345,7 @@ std::unique_ptr RunScenario( // Reduce channel count so that total channels specified is held regardless // of the number of clients available size_t num_channels = - (i == num_clients - 1) - ? channels_allocated - client_config.client_channels() - : client_config.client_channels() / num_clients; + (client_channels - channels_allocated) / (num_clients - i); channels_allocated += num_channels; per_client_config.set_client_channels(num_channels); From 5856dee7d6728bcdd0aaf8ac98abef4690252d4b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 22 Jun 2016 16:30:22 -0700 Subject: [PATCH 5/6] Fix compile --- test/cpp/qps/driver.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 6b31225f20a..b59c1efe127 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -31,6 +31,7 @@ * */ +#include #include #include #include @@ -345,9 +346,11 @@ std::unique_ptr RunScenario( // Reduce channel count so that total channels specified is held regardless // of the number of clients available size_t num_channels = - (client_channels - channels_allocated) / (num_clients - i); + (client_config.client_channels() - channels_allocated) / + (num_clients - i); channels_allocated += num_channels; - gpr_log(GPR_DEBUG, "Client %d gets %d channels", i, num_channels); + gpr_log(GPR_DEBUG, "Client %" PRIdPTR " gets %" PRIdPTR " channels", i, + num_channels); per_client_config.set_client_channels(num_channels); ClientArgs args; From bf3b769bd263047cc31ee9546ecc85396fca04a4 Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Thu, 11 Aug 2016 18:45:18 -0700 Subject: [PATCH 6/6] Tool for collecting build statistics. --- tools/run_tests/build_stats_schema.json | 56 +++++ .../build_stats_schema_no_matrix.json | 44 ++++ tools/run_tests/run_build_statistics.py | 204 ++++++++++++++++++ 3 files changed, 304 insertions(+) create mode 100644 tools/run_tests/build_stats_schema.json create mode 100644 tools/run_tests/build_stats_schema_no_matrix.json create mode 100755 tools/run_tests/run_build_statistics.py diff --git a/tools/run_tests/build_stats_schema.json b/tools/run_tests/build_stats_schema.json new file mode 100644 index 00000000000..021a3495452 --- /dev/null +++ b/tools/run_tests/build_stats_schema.json @@ -0,0 +1,56 @@ +[ + { + "name": "build_number", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "timestamp", + "type": "TIMESTAMP", + "mode": "NULLABLE" + }, + { + "name": "matrix", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "duration", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "pass_count", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "failure_count", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "error", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "description", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "count", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + } + ] + } +] diff --git a/tools/run_tests/build_stats_schema_no_matrix.json b/tools/run_tests/build_stats_schema_no_matrix.json new file mode 100644 index 00000000000..42650e30241 --- /dev/null +++ b/tools/run_tests/build_stats_schema_no_matrix.json @@ -0,0 +1,44 @@ +[ + { + "name": "build_number", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "timestamp", + "type": "TIMESTAMP", + "mode": "NULLABLE" + }, + { + "name": "duration", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "pass_count", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "failure_count", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "error", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "description", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "count", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + } +] diff --git a/tools/run_tests/run_build_statistics.py b/tools/run_tests/run_build_statistics.py new file mode 100755 index 00000000000..92c53782a8e --- /dev/null +++ b/tools/run_tests/run_build_statistics.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python2.7 +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Tool to get build statistics from Jenkins and upload to BigQuery.""" + +import argparse +import jenkinsapi +from jenkinsapi.custom_exceptions import JenkinsAPIException +from jenkinsapi.jenkins import Jenkins +import json +import os +import re +import sys +import urllib + + +gcp_utils_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../gcp/utils')) +sys.path.append(gcp_utils_dir) +import big_query_utils + + +_HAS_MATRIX=True +_PROJECT_ID = 'grpc-testing' +_HAS_MATRIX = True +_BUILDS = {'gRPC_master': _HAS_MATRIX, + 'gRPC_interop_master': not _HAS_MATRIX, + 'gRPC_pull_requests': _HAS_MATRIX, + 'gRPC_interop_pull_requests': not _HAS_MATRIX, +} +_URL_BASE = 'https://grpc-testing.appspot.com/job' +_KNOWN_ERRORS = [ + 'Failed to build workspace Tests with scheme AllTests', + 'Build timed out', + 'FATAL: Unable to produce a script file', + 'FAILED: Failed to build interop docker images', + 'LLVM ERROR: IO failure on output stream.', + 'MSBUILD : error MSB1009: Project file does not exist.', +] +_UNKNOWN_ERROR = 'Unknown error' +_DATASET_ID = 'build_statistics' + + +def _scrape_for_known_errors(html): + error_list = [] + known_error_count = 0 + for known_error in _KNOWN_ERRORS: + errors = re.findall(known_error, html) + this_error_count = len(errors) + if this_error_count > 0: + known_error_count += this_error_count + error_list.append({'description': known_error, + 'count': this_error_count}) + print('====> %d failures due to %s' % (this_error_count, known_error)) + return error_list, known_error_count + + +def _get_last_processed_buildnumber(build_name): + query = 'SELECT max(build_number) FROM [%s:%s.%s];' % ( + _PROJECT_ID, _DATASET_ID, build_name) + query_job = big_query_utils.sync_query_job(bq, _PROJECT_ID, query) + page = bq.jobs().getQueryResults( + pageToken=None, + **query_job['jobReference']).execute(num_retries=3) + if page['rows'][0]['f'][0]['v']: + return int(page['rows'][0]['f'][0]['v']) + return 0 + + +def _process_matrix(build, url_base): + matrix_list = [] + for matrix in build.get_matrix_runs(): + matrix_str = re.match('.*\\xc2\\xbb ((?:[^,]+,?)+) #.*', + matrix.name).groups()[0] + matrix_tuple = matrix_str.split(',') + json_url = '%s/config=%s,language=%s,platform=%s/testReport/api/json' % ( + url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2]) + console_url = '%s/config=%s,language=%s,platform=%s/consoleFull' % ( + url_base, matrix_tuple[0], matrix_tuple[1], matrix_tuple[2]) + matrix_dict = {'name': matrix_str, + 'duration': matrix.get_duration().total_seconds()} + matrix_dict.update(_process_build(json_url, console_url)) + matrix_list.append(matrix_dict) + + return matrix_list + + +def _process_build(json_url, console_url): + build_result = {} + error_list = [] + try: + html = urllib.urlopen(json_url).read() + test_result = json.loads(html) + print('====> Parsing result from %s' % json_url) + failure_count = test_result['failCount'] + build_result['pass_count'] = test_result['passCount'] + build_result['failure_count'] = failure_count + if failure_count > 0: + error_list, known_error_count = _scrape_for_known_errors(html) + unknown_error_count = failure_count - known_error_count + # This can happen if the same error occurs multiple times in one test. + if failure_count < known_error_count: + print('====> Some errors are duplicates.') + unknown_error_count = 0 + error_list.append({'description': _UNKNOWN_ERROR, + 'count': unknown_error_count}) + except Exception as e: + print('====> Got exception for %s: %s.' % (json_url, str(e))) + print('====> Parsing errors from %s.' % console_url) + html = urllib.urlopen(console_url).read() + build_result['pass_count'] = 0 + build_result['failure_count'] = 1 + error_list, _ = _scrape_for_known_errors(html) + if error_list: + error_list.append({'description': _UNKNOWN_ERROR, 'count': 0}) + else: + error_list.append({'description': _UNKNOWN_ERROR, 'count': 1}) + + if error_list: + build_result['error'] = error_list + + return build_result + + +# parse command line +argp = argparse.ArgumentParser(description='Get build statistics.') +argp.add_argument('-u', '--username', default='jenkins') +argp.add_argument('-b', '--builds', + choices=['all'] + sorted(_BUILDS.keys()), + nargs='+', + default=['all']) +args = argp.parse_args() + +J = Jenkins('https://grpc-testing.appspot.com', args.username, 'apiToken') +bq = big_query_utils.create_big_query() + +for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds: + print('====> Build: %s' % build_name) + # Since get_last_completed_build() always fails due to malformatted string + # error, we use get_build_metadata() instead. + job = None + try: + job = J[build_name] + except Exception as e: + print('====> Failed to get build %s: %s.' % (build_name, str(e))) + continue + last_processed_build_number = _get_last_processed_buildnumber(build_name) + last_complete_build_number = job.get_last_completed_buildnumber() + # To avoid processing all builds for a project never looked at. In this case, + # only examine 10 latest builds. + starting_build_number = max(last_processed_build_number+1, + last_complete_build_number-9) + for build_number in xrange(starting_build_number, + last_complete_build_number+1): + print('====> Processing %s build %d.' % (build_name, build_number)) + build = None + try: + build = job.get_build_metadata(build_number) + except KeyError: + print('====> Build %s is missing. Skip.' % build_number) + continue + build_result = {'build_number': build_number, + 'timestamp': str(build.get_timestamp())} + url_base = json_url = '%s/%s/%d' % (_URL_BASE, build_name, build_number) + if _BUILDS[build_name]: # The build has matrix, such as gRPC_master. + build_result['matrix'] = _process_matrix(build, url_base) + else: + json_url = '%s/testReport/api/json' % url_base + console_url = '%s/consoleFull' % url_base + build_result['duration'] = build.get_duration().total_seconds() + build_result.update(_process_build(json_url, console_url)) + rows = [big_query_utils.make_row(build_number, build_result)] + if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name, + rows): + print '====> Error uploading result to bigquery.' + sys.exit(1) +