From 1898e1be256fe4796f2ba6f0b9e2bf7b774f460a Mon Sep 17 00:00:00 2001 From: Bill Feng Date: Thu, 14 Feb 2019 10:39:47 -0800 Subject: [PATCH 1/8] debug printouts --- tools/run_tests/python_utils/upload_rbe_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/python_utils/upload_rbe_results.py b/tools/run_tests/python_utils/upload_rbe_results.py index 11cc1aa5dd4..ba2c012e20f 100755 --- a/tools/run_tests/python_utils/upload_rbe_results.py +++ b/tools/run_tests/python_utils/upload_rbe_results.py @@ -145,6 +145,7 @@ if __name__ == "__main__": api_key = args.api_key or _get_api_key() invocation_id = args.invocation_id or _get_invocation_id() resultstore_actions = _get_resultstore_data(api_key, invocation_id) + print(resultstore_actions) bq_rows = [] for index, action in enumerate(resultstore_actions): From 5428bdeb8388abc5a9601a298c20924730cd245f Mon Sep 17 00:00:00 2001 From: Bill Feng Date: Thu, 14 Feb 2019 11:09:55 -0800 Subject: [PATCH 2/8] more debug printouts --- tools/run_tests/python_utils/upload_rbe_results.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/python_utils/upload_rbe_results.py b/tools/run_tests/python_utils/upload_rbe_results.py index ba2c012e20f..0dfa487033a 100755 --- a/tools/run_tests/python_utils/upload_rbe_results.py +++ b/tools/run_tests/python_utils/upload_rbe_results.py @@ -96,8 +96,10 @@ def _upload_results_to_bq(rows): max_retries = 3 for attempt in range(max_retries): - if big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, _TABLE_ID, - rows): + k = big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, _TABLE_ID, + rows) + print(k) + if k: break else: if attempt < max_retries - 1: @@ -145,7 +147,6 @@ if __name__ == "__main__": api_key = args.api_key or _get_api_key() invocation_id = args.invocation_id or _get_invocation_id() resultstore_actions = _get_resultstore_data(api_key, invocation_id) - print(resultstore_actions) bq_rows = [] for index, action in enumerate(resultstore_actions): From 4d0b1236092eb1e2484b771420eae79ef380904e Mon Sep 17 00:00:00 2001 From: Bill Feng Date: Thu, 14 Feb 2019 13:52:43 -0800 Subject: [PATCH 3/8] more debug printouts --- tools/gcp/utils/big_query_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/gcp/utils/big_query_utils.py b/tools/gcp/utils/big_query_utils.py index 6e9cda376ba..339ad0f9f98 100755 --- a/tools/gcp/utils/big_query_utils.py +++ b/tools/gcp/utils/big_query_utils.py @@ -178,6 +178,7 @@ def insert_rows(big_query, project_id, dataset_id, table_id, rows_list): is_success = False except HttpError as http_error: print('Error inserting rows to the table %s' % table_id) + print(HttpError) is_success = False return is_success From cd83999dd916dbaf845022ea9138b149cb154df7 Mon Sep 17 00:00:00 2001 From: Bill Feng Date: Thu, 14 Feb 2019 14:22:52 -0800 Subject: [PATCH 4/8] debug outputs --- tools/gcp/utils/big_query_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/gcp/utils/big_query_utils.py b/tools/gcp/utils/big_query_utils.py index 339ad0f9f98..1f675a79e9c 100755 --- a/tools/gcp/utils/big_query_utils.py +++ b/tools/gcp/utils/big_query_utils.py @@ -178,7 +178,7 @@ def insert_rows(big_query, project_id, dataset_id, table_id, rows_list): is_success = False except HttpError as http_error: print('Error inserting rows to the table %s' % table_id) - print(HttpError) + print(http_error) is_success = False return is_success From be55d61b6485770ea73a6947af43523e9b2b0b56 Mon Sep 17 00:00:00 2001 From: Bill Feng Date: Thu, 14 Feb 2019 15:07:04 -0800 Subject: [PATCH 5/8] more fields for filter --- tools/gcp/utils/big_query_utils.py | 2 +- tools/run_tests/python_utils/upload_rbe_results.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/gcp/utils/big_query_utils.py b/tools/gcp/utils/big_query_utils.py index 1f675a79e9c..168b48d1e65 100755 --- a/tools/gcp/utils/big_query_utils.py +++ b/tools/gcp/utils/big_query_utils.py @@ -178,7 +178,7 @@ def insert_rows(big_query, project_id, dataset_id, table_id, rows_list): is_success = False except HttpError as http_error: print('Error inserting rows to the table %s' % table_id) - print(http_error) + print('Error message: %s' % http_error) is_success = False return is_success diff --git a/tools/run_tests/python_utils/upload_rbe_results.py b/tools/run_tests/python_utils/upload_rbe_results.py index 0dfa487033a..3dd384efd91 100755 --- a/tools/run_tests/python_utils/upload_rbe_results.py +++ b/tools/run_tests/python_utils/upload_rbe_results.py @@ -124,7 +124,7 @@ def _get_resultstore_data(api_key, invocation_id): while True: req = urllib2.Request( url= - 'https://resultstore.googleapis.com/v2/invocations/%s/targets/-/configuredTargets/-/actions?key=%s&pageToken=%s&fields=next_page_token,actions.id,actions.status_attributes' + 'https://resultstore.googleapis.com/v2/invocations/%s/targets/-/configuredTargets/-/actions?key=%s&pageToken=%s&fields=next_page_token,actions.id,actions.status_attributes,actions.timing,actions.test_action' % (invocation_id, api_key, page_token), headers={ 'Content-Type': 'application/json' From 4564b78b2724404252166d2921d726e6b409e06d Mon Sep 17 00:00:00 2001 From: Bill Feng Date: Thu, 14 Feb 2019 15:29:19 -0800 Subject: [PATCH 6/8] more fields for filter --- tools/run_tests/python_utils/upload_rbe_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/python_utils/upload_rbe_results.py b/tools/run_tests/python_utils/upload_rbe_results.py index 3dd384efd91..6eb47356bc9 100755 --- a/tools/run_tests/python_utils/upload_rbe_results.py +++ b/tools/run_tests/python_utils/upload_rbe_results.py @@ -150,6 +150,7 @@ if __name__ == "__main__": bq_rows = [] for index, action in enumerate(resultstore_actions): + print(action); # Filter out non-test related data, such as build results. if 'testAction' not in action: continue From 43277f83c8f061d01fcb68b040391a653308c670 Mon Sep 17 00:00:00 2001 From: Bill Feng Date: Thu, 14 Feb 2019 16:29:08 -0800 Subject: [PATCH 7/8] filted out malformed resultstore data --- tools/run_tests/python_utils/upload_rbe_results.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/run_tests/python_utils/upload_rbe_results.py b/tools/run_tests/python_utils/upload_rbe_results.py index 6eb47356bc9..615313f7ea4 100755 --- a/tools/run_tests/python_utils/upload_rbe_results.py +++ b/tools/run_tests/python_utils/upload_rbe_results.py @@ -188,6 +188,8 @@ if __name__ == "__main__": 'startTime': resultstore_actions[index - 1]['timing']['startTime'] } + elif 'testSuite' not in action['testAction']: + continue else: test_cases = action['testAction']['testSuite']['tests'][0][ 'testSuite']['tests'] From 987ac52ee2b6e17de79306defa2a05ae7ba04868 Mon Sep 17 00:00:00 2001 From: Bill Feng Date: Thu, 14 Feb 2019 17:46:41 -0800 Subject: [PATCH 8/8] formatting --- tools/run_tests/python_utils/upload_rbe_results.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/python_utils/upload_rbe_results.py b/tools/run_tests/python_utils/upload_rbe_results.py index 615313f7ea4..bda567e977e 100755 --- a/tools/run_tests/python_utils/upload_rbe_results.py +++ b/tools/run_tests/python_utils/upload_rbe_results.py @@ -96,10 +96,8 @@ def _upload_results_to_bq(rows): max_retries = 3 for attempt in range(max_retries): - k = big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, _TABLE_ID, - rows) - print(k) - if k: + if big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, _TABLE_ID, + rows): break else: if attempt < max_retries - 1: @@ -150,7 +148,6 @@ if __name__ == "__main__": bq_rows = [] for index, action in enumerate(resultstore_actions): - print(action); # Filter out non-test related data, such as build results. if 'testAction' not in action: continue