Add retries for check_on_pr

tools/run_tests/python_utils/check_on_pr.py

@@ -14,6 +14,7 @@
 
 from __future__ import print_function
 import os
+import sys
 import json
 import time
 import datetime
@@ -27,6 +28,8 @@ _GITHUB_APP_ID = 22338
 _INSTALLATION_ID = 519109
 
 _ACCESS_TOKEN_CACHE = None
+_ACCESS_TOKEN_FETCH_RETRIES = 5
+_ACCESS_TOKEN_FETCH_RETRIES_INTERVAL_S = 1
 
 
 def _jwt_token():
@@ -46,13 +49,28 @@ def _jwt_token():
 def _access_token():
     global _ACCESS_TOKEN_CACHE
     if _ACCESS_TOKEN_CACHE == None or _ACCESS_TOKEN_CACHE['exp'] < time.time():
-        resp = requests.post(
-            url='https://api.github.com/app/installations/%s/access_tokens' %
-            _INSTALLATION_ID,
-            headers={
-                'Authorization': 'Bearer %s' % _jwt_token().decode('ASCII'),
-                'Accept': 'application/vnd.github.machine-man-preview+json',
-            })
+        for i in range(_ACCESS_TOKEN_FETCH_RETRIES):
+            resp = requests.post(
+                url='https://api.github.com/app/installations/%s/access_tokens'
+                % _INSTALLATION_ID,
+                headers={
+                    'Authorization': 'Bearer %s' % _jwt_token().decode('ASCII'),
+                    'Accept': 'application/vnd.github.machine-man-preview+json',
+                })
+            if resp.status_code == 200:
+                break
+            else:
+                print("Fetch access token from Github API failed:")
+                print(resp.json())
+                if i != _ACCESS_TOKEN_FETCH_RETRIES - 1:
+                    print('Retrying after %.2f second.' %
+                          _ACCESS_TOKEN_FETCH_RETRIES_INTERVAL_S)
+                    time.sleep(_ACCESS_TOKEN_FETCH_RETRIES_INTERVAL_S)
+
+        if resp.status_code != 200:
+            print("error: Unable to fetch access token, exiting...")
+            sys.exit(1)
+
         _ACCESS_TOKEN_CACHE = {
             'token': resp.json()['token'],
             'exp': time.time() + 60