From cc3780eb506d34b884bb4f71a1e1ad39c4050331 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sun, 26 Aug 2018 20:30:28 +0200 Subject: [PATCH 1/3] refactor upload_to_bq functions --- .../interop_matrix/run_interop_matrix_tests.py | 2 +- .../python_utils/upload_test_results.py | 17 +++++------------ tools/run_tests/run_interop_tests.py | 2 +- tools/run_tests/run_tests.py | 9 +++++++-- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py index 9d442346a70..6cd6f431671 100755 --- a/tools/interop_matrix/run_interop_matrix_tests.py +++ b/tools/interop_matrix/run_interop_matrix_tests.py @@ -235,7 +235,7 @@ def run_tests_for_lang(lang, runtime, images): maxjobs=args.jobs) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( - resultset, args.bq_result_table, args) + resultset, args.bq_result_table) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py index 9d997037259..f04ef37e087 100644 --- a/tools/run_tests/python_utils/upload_test_results.py +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -104,14 +104,13 @@ def _insert_rows_with_retries(bq, bq_table, bq_rows): sys.exit(1) -def upload_results_to_bq(resultset, bq_table, args, platform): +def upload_results_to_bq(resultset, bq_table, extra_fields): """Upload test results to a BQ table. Args: resultset: dictionary generated by jobset.run bq_table: string name of table to create/upload results to in BQ - args: args in run_tests.py, generated by argparse - platform: string name of platform tests were run on + extra_fields: dict with extra values that will be uploaded along with the results """ bq = big_query_utils.create_big_query() big_query_utils.create_partitioned_table( @@ -129,32 +128,26 @@ def upload_results_to_bq(resultset, bq_table, args, platform): for result in results: test_results = {} _get_build_metadata(test_results) - test_results['compiler'] = args.compiler - test_results['config'] = args.config test_results['cpu_estimated'] = result.cpu_estimated test_results['cpu_measured'] = result.cpu_measured test_results['elapsed_time'] = '%.2f' % result.elapsed_time - test_results['iomgr_platform'] = args.iomgr_platform - # args.language is a list, but will always have one element in the contexts - # this function is used. - test_results['language'] = args.language[0] - test_results['platform'] = platform test_results['result'] = result.state test_results['return_code'] = result.returncode test_results['test_name'] = shortname test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') + for field_name, field_value in extra_fields: + test_results[field_name] = field_value row = big_query_utils.make_row(str(uuid.uuid4()), test_results) bq_rows.append(row) _insert_rows_with_retries(bq, bq_table, bq_rows) -def upload_interop_results_to_bq(resultset, bq_table, args): +def upload_interop_results_to_bq(resultset, bq_table): """Upload interop test results to a BQ table. Args: resultset: dictionary generated by jobset.run bq_table: string name of table to create/upload results to in BQ - args: args in run_interop_tests.py, generated by argparse """ bq = big_query_utils.create_big_query() big_query_utils.create_partitioned_table( diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 22055d58e8b..2936bdfa802 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -1494,7 +1494,7 @@ try: maxjobs=args.jobs, skip_jobs=args.manual_run) if args.bq_result_table and resultset: - upload_interop_results_to_bq(resultset, args.bq_result_table, args) + upload_interop_results_to_bq(resultset, args.bq_result_table) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) else: diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index e04b13b24ca..c4954df12b5 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1821,8 +1821,13 @@ def _build_and_run(check_cancelled, for antagonist in antagonists: antagonist.kill() if args.bq_result_table and resultset: - upload_results_to_bq(resultset, args.bq_result_table, args, - platform_string()) + upload_extra_fields={ + 'compiler': args.compiler, + 'config': args.config, + 'iomgr_plaform': args.iomgr_platform, + 'language': args.language[0], # args.language is a list but will always have one element when uploading to BQ is enabled. + 'plaform': platform_string()} + upload_results_to_bq(resultset, args.bq_result_table, upload_extra_fields) if xml_report and resultset: report_utils.render_junit_xml_report( resultset, xml_report, suite_name=args.report_suite_name) From 3c78606fb22d4cb9f648165688285ecfa00b7725 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 30 Aug 2018 18:45:38 +0200 Subject: [PATCH 2/3] yapf code --- tools/run_tests/run_tests.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index c4954df12b5..60c35b7220d 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1821,13 +1821,16 @@ def _build_and_run(check_cancelled, for antagonist in antagonists: antagonist.kill() if args.bq_result_table and resultset: - upload_extra_fields={ + upload_extra_fields = { 'compiler': args.compiler, 'config': args.config, 'iomgr_plaform': args.iomgr_platform, - 'language': args.language[0], # args.language is a list but will always have one element when uploading to BQ is enabled. - 'plaform': platform_string()} - upload_results_to_bq(resultset, args.bq_result_table, upload_extra_fields) + 'language': args.language[ + 0], # args.language is a list but will always have one element when uploading to BQ is enabled. + 'plaform': platform_string() + } + upload_results_to_bq(resultset, args.bq_result_table, + upload_extra_fields) if xml_report and resultset: report_utils.render_junit_xml_report( resultset, xml_report, suite_name=args.report_suite_name) From af1966aa037bf3515f553aee87c886ac829efc02 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 30 Aug 2018 22:26:56 +0200 Subject: [PATCH 3/3] fixes to run_tests.py bq uploading --- tools/run_tests/python_utils/upload_test_results.py | 2 +- tools/run_tests/run_tests.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py index f04ef37e087..0ca23f56cf1 100644 --- a/tools/run_tests/python_utils/upload_test_results.py +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -135,7 +135,7 @@ def upload_results_to_bq(resultset, bq_table, extra_fields): test_results['return_code'] = result.returncode test_results['test_name'] = shortname test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') - for field_name, field_value in extra_fields: + for field_name, field_value in six.iteritems(extra_fields): test_results[field_name] = field_value row = big_query_utils.make_row(str(uuid.uuid4()), test_results) bq_rows.append(row) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 60c35b7220d..ecb5e1d899e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1824,10 +1824,10 @@ def _build_and_run(check_cancelled, upload_extra_fields = { 'compiler': args.compiler, 'config': args.config, - 'iomgr_plaform': args.iomgr_platform, + 'iomgr_platform': args.iomgr_platform, 'language': args.language[ 0], # args.language is a list but will always have one element when uploading to BQ is enabled. - 'plaform': platform_string() + 'platform': platform_string() } upload_results_to_bq(resultset, args.bq_result_table, upload_extra_fields)