From cc42e6dba8035ec8dd77c59d6c13f9d510e6cf14 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 11 Jun 2018 11:53:49 -0400 Subject: [PATCH] upload bq test results in batches --- tools/run_tests/python_utils/upload_test_results.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py index 09dcd57ad4c..63fa38b6789 100644 --- a/tools/run_tests/python_utils/upload_test_results.py +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -163,6 +163,7 @@ def upload_interop_results_to_bq(resultset, bq_table, args): expiration_ms=_EXPIRATION_MS) for shortname, results in six.iteritems(resultset): + bq_rows = [] for result in results: test_results = {} _get_build_metadata(test_results) @@ -175,11 +176,15 @@ def upload_interop_results_to_bq(resultset, bq_table, args): test_results['test_case'] = shortname.split(':')[3] test_results['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S') row = big_query_utils.make_row(str(uuid.uuid4()), test_results) - # TODO(jtattermusch): rows are inserted one by one, very inefficient + bq_rows.append(row) + + # BigQuery sometimes fails with large uploads, so batch 1,000 rows at a time. + for i in range((len(bq_rows) / 1000) + 1): max_retries = 3 for attempt in range(max_retries): - if big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, - bq_table, [row]): + if big_query_utils.insert_rows( + bq, _PROJECT_ID, _DATASET_ID, bq_table, + bq_rows[i * 1000:(i + 1) * 1000]): break else: if attempt < max_retries - 1: