do best-effort cleanup even if some resources not found (#26248)

* do best-effort cleanup even if some resources not found

* log deletions

* fix log

* yapf

tools/run_tests/run_xds_tests.py

@@ -2537,62 +2537,96 @@ def create_global_forwarding_rule(gcp,
 
 
 def get_health_check(gcp, health_check_name):
-    result = gcp.compute.healthChecks().get(
-        project=gcp.project, healthCheck=health_check_name).execute()
-    gcp.health_check = GcpResource(health_check_name, result['selfLink'])
+    try:
+        result = gcp.compute.healthChecks().get(
+            project=gcp.project, healthCheck=health_check_name).execute()
+        gcp.health_check = GcpResource(health_check_name, result['selfLink'])
+    except Exception as e:
+        gcp.errors.append(e)
+        gcp.health_check = GcpResource(health_check_name, None)
 
 
 def get_health_check_firewall_rule(gcp, firewall_name):
-    result = gcp.compute.firewalls().get(project=gcp.project,
-                                         firewall=firewall_name).execute()
-    gcp.health_check_firewall_rule = GcpResource(firewall_name,
-                                                 result['selfLink'])
+    try:
+        result = gcp.compute.firewalls().get(project=gcp.project,
+                                             firewall=firewall_name).execute()
+        gcp.health_check_firewall_rule = GcpResource(firewall_name,
+                                                     result['selfLink'])
+    except Exception as e:
+        gcp.errors.append(e)
+        gcp.health_check_firewall_rule = GcpResource(firewall_name, None)
 
 
 def get_backend_service(gcp, backend_service_name):
-    result = gcp.compute.backendServices().get(
-        project=gcp.project, backendService=backend_service_name).execute()
-    backend_service = GcpResource(backend_service_name, result['selfLink'])
+    try:
+        result = gcp.compute.backendServices().get(
+            project=gcp.project, backendService=backend_service_name).execute()
+        backend_service = GcpResource(backend_service_name, result['selfLink'])
+    except Exception as e:
+        gcp.errors.append(e)
+        backend_service = GcpResource(backend_service_name, None)
     gcp.backend_services.append(backend_service)
     return backend_service
 
 
 def get_url_map(gcp, url_map_name):
-    result = gcp.compute.urlMaps().get(project=gcp.project,
-                                       urlMap=url_map_name).execute()
-    gcp.url_map = GcpResource(url_map_name, result['selfLink'])
+    try:
+        result = gcp.compute.urlMaps().get(project=gcp.project,
+                                           urlMap=url_map_name).execute()
+        gcp.url_map = GcpResource(url_map_name, result['selfLink'])
+    except Exception as e:
+        gcp.errors.append(e)
+        gcp.url_map = GcpResource(url_map_name, None)
 
 
 def get_target_proxy(gcp, target_proxy_name):
-    if gcp.alpha_compute:
-        result = gcp.alpha_compute.targetGrpcProxies().get(
-            project=gcp.project, targetGrpcProxy=target_proxy_name).execute()
-    else:
-        result = gcp.compute.targetHttpProxies().get(
-            project=gcp.project, targetHttpProxy=target_proxy_name).execute()
-    gcp.target_proxy = GcpResource(target_proxy_name, result['selfLink'])
+    try:
+        if gcp.alpha_compute:
+            result = gcp.alpha_compute.targetGrpcProxies().get(
+                project=gcp.project,
+                targetGrpcProxy=target_proxy_name).execute()
+        else:
+            result = gcp.compute.targetHttpProxies().get(
+                project=gcp.project,
+                targetHttpProxy=target_proxy_name).execute()
+        gcp.target_proxy = GcpResource(target_proxy_name, result['selfLink'])
+    except Exception as e:
+        gcp.errors.append(e)
+        gcp.target_proxy = GcpResource(target_proxy_name, None)
 
 
 def get_global_forwarding_rule(gcp, forwarding_rule_name):
-    result = gcp.compute.globalForwardingRules().get(
-        project=gcp.project, forwardingRule=forwarding_rule_name).execute()
-    gcp.global_forwarding_rule = GcpResource(forwarding_rule_name,
-                                             result['selfLink'])
+    try:
+        result = gcp.compute.globalForwardingRules().get(
+            project=gcp.project, forwardingRule=forwarding_rule_name).execute()
+        gcp.global_forwarding_rule = GcpResource(forwarding_rule_name,
+                                                 result['selfLink'])
+    except Exception as e:
+        gcp.errors.append(e)
+        gcp.global_forwarding_rule = GcpResource(forwarding_rule_name, None)
 
 
 def get_instance_template(gcp, template_name):
-    result = gcp.compute.instanceTemplates().get(
-        project=gcp.project, instanceTemplate=template_name).execute()
-    gcp.instance_template = GcpResource(template_name, result['selfLink'])
+    try:
+        result = gcp.compute.instanceTemplates().get(
+            project=gcp.project, instanceTemplate=template_name).execute()
+        gcp.instance_template = GcpResource(template_name, result['selfLink'])
+    except Exception as e:
+        gcp.errors.append(e)
+        gcp.instance_template = GcpResource(template_name, None)
 
 
 def get_instance_group(gcp, zone, instance_group_name):
-    result = gcp.compute.instanceGroups().get(
-        project=gcp.project, zone=zone,
-        instanceGroup=instance_group_name).execute()
-    gcp.service_port = result['namedPorts'][0]['port']
-    instance_group = InstanceGroup(instance_group_name, result['selfLink'],
-                                   zone)
+    try:
+        result = gcp.compute.instanceGroups().get(
+            project=gcp.project, zone=zone,
+            instanceGroup=instance_group_name).execute()
+        gcp.service_port = result['namedPorts'][0]['port']
+        instance_group = InstanceGroup(instance_group_name, result['selfLink'],
+                                       zone)
+    except Exception as e:
+        gcp.errors.append(e)
+        instance_group = InstanceGroup(instance_group_name, None, zone)
     gcp.instance_groups.append(instance_group)
     return instance_group
 
@@ -2603,6 +2637,7 @@ def delete_global_forwarding_rule(gcp, name=None):
     else:
         forwarding_rule_to_delete = gcp.global_forwarding_rule.name
     try:
+        logger.debug('Deleting forwarding rule %s', forwarding_rule_to_delete)
         result = gcp.compute.globalForwardingRules().delete(
             project=gcp.project,
             forwardingRule=forwarding_rule_to_delete).execute(
@@ -2619,10 +2654,12 @@ def delete_target_proxy(gcp, name=None):
         proxy_to_delete = gcp.target_proxy.name
     try:
         if gcp.alpha_compute:
+            logger.debug('Deleting grpc proxy %s', proxy_to_delete)
             result = gcp.alpha_compute.targetGrpcProxies().delete(
                 project=gcp.project, targetGrpcProxy=proxy_to_delete).execute(
                     num_retries=_GCP_API_RETRIES)
         else:
+            logger.debug('Deleting http proxy %s', proxy_to_delete)
             result = gcp.compute.targetHttpProxies().delete(
                 project=gcp.project, targetHttpProxy=proxy_to_delete).execute(
                     num_retries=_GCP_API_RETRIES)
@@ -2637,6 +2674,7 @@ def delete_url_map(gcp, name=None):
     else:
         url_map_to_delete = gcp.url_map.name
     try:
+        logger.debug('Deleting url map %s', url_map_to_delete)
         result = gcp.compute.urlMaps().delete(
             project=gcp.project,
             urlMap=url_map_to_delete).execute(num_retries=_GCP_API_RETRIES)
@@ -2647,6 +2685,7 @@ def delete_url_map(gcp, name=None):
 
 def delete_backend_service(gcp, backend_service):
     try:
+        logger.debug('Deleting backend service %s', backend_service.name)
         result = gcp.compute.backendServices().delete(
             project=gcp.project, backendService=backend_service.name).execute(
                 num_retries=_GCP_API_RETRIES)
@@ -2662,6 +2701,8 @@ def delete_backend_services(gcp):
 
 def delete_firewall(gcp):
     try:
+        logger.debug('Deleting firewall %s',
+                     gcp.health_check_firewall_rule.name)
         result = gcp.compute.firewalls().delete(
             project=gcp.project,
             firewall=gcp.health_check_firewall_rule.name).execute(
@@ -2673,6 +2714,7 @@ def delete_firewall(gcp):
 
 def delete_health_check(gcp):
     try:
+        logger.debug('Deleting health check %s', gcp.health_check.name)
         result = gcp.compute.healthChecks().delete(
             project=gcp.project, healthCheck=gcp.health_check.name).execute(
                 num_retries=_GCP_API_RETRIES)
@@ -2684,6 +2726,8 @@ def delete_health_check(gcp):
 def delete_instance_groups(gcp):
     for instance_group in gcp.instance_groups:
         try:
+            logger.debug('Deleting instance group %s %s', instance_group.name,
+                         instance_group.zone)
             result = gcp.compute.instanceGroupManagers().delete(
                 project=gcp.project,
                 zone=instance_group.zone,
@@ -2699,6 +2743,8 @@ def delete_instance_groups(gcp):
 
 def delete_instance_template(gcp):
     try:
+        logger.debug('Deleting instance template %s',
+                     gcp.instance_template.name)
         result = gcp.compute.instanceTemplates().delete(
             project=gcp.project,
             instanceTemplate=gcp.instance_template.name).execute(
@@ -2948,6 +2994,7 @@ class GcpState(object):
         self.service_port = None
         self.instance_template = None
         self.instance_groups = []
+        self.errors = []
 
 
 alpha_compute = None
@@ -3005,13 +3052,7 @@ try:
     if args.use_existing_gcp_resources:
         logger.info('Reusing existing GCP resources')
         get_health_check(gcp, health_check_name)
-        try:
-            get_health_check_firewall_rule(gcp, firewall_name)
-        except googleapiclient.errors.HttpError as http_error:
-            # Firewall rule may be auto-deleted periodically depending on GCP
-            # project settings.
-            logger.exception('Failed to find firewall rule, recreating')
-            create_health_check_firewall_rule(gcp, firewall_name)
+        get_health_check_firewall_rule(gcp, firewall_name)
         backend_service = get_backend_service(gcp, backend_service_name)
         alternate_backend_service = get_backend_service(
             gcp, alternate_backend_service_name)
@@ -3024,6 +3065,8 @@ try:
             gcp, args.zone, same_zone_instance_group_name)
         secondary_zone_instance_group = get_instance_group(
             gcp, args.secondary_zone, secondary_zone_instance_group_name)
+        if gcp.errors:
+            raise Exception(gcp.errors)
     else:
         create_health_check_firewall_rule(gcp, firewall_name)
         backend_service = add_backend_service(gcp, backend_service_name)