Merge pull request #10115 from thunderboltsid/update-run-tests-scripts-to-python3.x

Make testing scripts python3.x compatible.

tools/run_tests/artifacts/artifact_targets.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2016, Google Inc.
 # All rights reserved.
 #

tools/run_tests/artifacts/distribtest_targets.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2016, Google Inc.
 # All rights reserved.
 #

tools/run_tests/artifacts/package_targets.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2016, Google Inc.
 # All rights reserved.
 #

tools/run_tests/performance/bq_upload_result.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2016, Google Inc.
 # All rights reserved.
 #
@@ -30,6 +30,8 @@
 
 # Uploads performance benchmark result file to bigquery.
 
+from __future__ import print_function
+
 import argparse
 import calendar
 import json
@@ -70,7 +72,7 @@ def _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, result_file):
   _create_results_table(bq, dataset_id, table_id)
 
   if not _insert_result(bq, dataset_id, table_id, scenario_result, flatten=False):
-    print 'Error uploading result to bigquery.'
+    print('Error uploading result to bigquery.')
     sys.exit(1)
 
 
@@ -82,7 +84,7 @@ def _upload_scenario_result_to_bigquery(dataset_id, table_id, result_file):
   _create_results_table(bq, dataset_id, table_id)
 
   if not _insert_result(bq, dataset_id, table_id, scenario_result):
-    print 'Error uploading result to bigquery.'
+    print('Error uploading result to bigquery.')
     sys.exit(1)
 
 
@@ -179,4 +181,4 @@ if args.file_format == 'netperf_latency_csv':
   _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, args.file_to_upload)
 else:
   _upload_scenario_result_to_bigquery(dataset_id, table_id, args.file_to_upload)
-print 'Successfully uploaded %s to BigQuery.\n' % args.file_to_upload
+print('Successfully uploaded %s to BigQuery.\n' % args.file_to_upload)

tools/run_tests/python_utils/antagonist.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #

tools/run_tests/python_utils/filter_pull_request_tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -30,7 +30,10 @@
 
 """Filter out tests based on file differences compared to merge target branch"""
 
+from __future__ import print_function
+
 import re
+import six
 from subprocess import check_output
 
 
@@ -125,7 +128,7 @@ _WHITELIST_DICT = {
 }
 
 # Add all triggers to their respective test suites
-for trigger, test_suites in _WHITELIST_DICT.iteritems():
+for trigger, test_suites in six.iteritems(_WHITELIST_DICT):
   for test_suite in test_suites:
     test_suite.add_trigger(trigger)
 

tools/run_tests/python_utils/port_server.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #

tools/run_tests/python_utils/report_utils.py

@@ -40,6 +40,7 @@ except (ImportError):
 import os
 import string
 import xml.etree.cElementTree as ET
+import six
 
 
 def _filter_msg(msg, output_format):
@@ -63,7 +64,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc',
   root = ET.Element('testsuites')
   testsuite = ET.SubElement(root, 'testsuite', id='1', package=suite_package,
                             name=suite_name)
-  for shortname, results in resultset.iteritems():
+  for shortname, results in six.iteritems(resultset):
     for result in results:
       xml_test = ET.SubElement(testsuite, 'testcase', name=shortname)
       if result.elapsed_time:

tools/run_tests/run_build_statistics.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2016, Google Inc.
 # All rights reserved.
 #
@@ -30,6 +30,8 @@
 
 """Tool to get build statistics from Jenkins and upload to BigQuery."""
 
+from __future__ import print_function
+
 import argparse
 import jenkinsapi
 from jenkinsapi.custom_exceptions import JenkinsAPIException
@@ -243,6 +245,6 @@ for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
     rows = [big_query_utils.make_row(build_number, build_result)]
     if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name, 
                                        rows):
-      print '====> Error uploading result to bigquery.'
+      print('====> Error uploading result to bigquery.')
       sys.exit(1)
 

tools/run_tests/run_interop_tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -44,6 +44,7 @@ import sys
 import tempfile
 import time
 import uuid
+import six
 
 import python_utils.dockerjob as dockerjob
 import python_utils.jobset as jobset
@@ -885,7 +886,7 @@ if not args.use_docker and servers:
 
 languages = set(_LANGUAGES[l]
                 for l in itertools.chain.from_iterable(
-                    _LANGUAGES.iterkeys() if x == 'all' else [x]
+                    six.iterkeys(_LANGUAGES) if x == 'all' else [x]
                     for x in args.language))
 
 languages_http2_badserver_interop = set()
@@ -925,7 +926,7 @@ if args.use_docker:
     else:
       jobset.message('FAILED', 'Failed to build interop docker images.',
                      do_newline=True)
-      for image in docker_images.itervalues():
+      for image in six.itervalues(docker_images):
         dockerjob.remove_image(image, skip_nonexistent=True)
       sys.exit(1)
 
@@ -1053,7 +1054,7 @@ try:
 
   if not jobs:
     print('No jobs to run.')
-    for image in docker_images.itervalues():
+    for image in six.itervalues(docker_images):
       dockerjob.remove_image(image, skip_nonexistent=True)
     sys.exit(1)
 
@@ -1093,9 +1094,9 @@ finally:
     if not job.is_running():
       print('Server "%s" has exited prematurely.' % server)
 
-  dockerjob.finish_jobs([j for j in server_jobs.itervalues()])
+  dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)])
 
-  for image in docker_images.itervalues():
+  for image in six.itervalues(docker_images):
     if not args.manual_run:
       print('Removing docker image %s' % image)
       dockerjob.remove_image(image)

tools/run_tests/run_microbenchmark.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2017, Google Inc.
 # All rights reserved.
 #

tools/run_tests/run_performance_tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2016, Google Inc.
 # All rights reserved.
 #
@@ -46,6 +46,7 @@ import tempfile
 import time
 import traceback
 import uuid
+import six
 
 import performance.scenario_config as scenario_config
 import python_utils.jobset as jobset
@@ -502,8 +503,8 @@ args = argp.parse_args()
 
 languages = set(scenario_config.LANGUAGES[l]
                 for l in itertools.chain.from_iterable(
-                      scenario_config.LANGUAGES.iterkeys() if x == 'all' else [x]
-                      for x in args.language))
+                      six.iterkeys(scenario_config.LANGUAGES) if x == 'all'
+                      else [x] for x in args.language))
 
 
 # Put together set of remote hosts where to run and build
@@ -572,8 +573,8 @@ for scenario in scenarios:
         jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host))
       scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1)
       total_scenario_failures += scenario_failures
-      merged_resultset = dict(itertools.chain(merged_resultset.iteritems(),
-                                              resultset.iteritems()))
+      merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset),
+                                              six.iteritems(resultset)))
     finally:
       # Consider qps workers that need to be killed as failures
       qps_workers_killed += finish_qps_workers(scenario.workers)

tools/run_tests/run_stress_tests.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -43,6 +43,7 @@ import sys
 import tempfile
 import time
 import uuid
+import six
 
 import python_utils.dockerjob as dockerjob
 import python_utils.jobset as jobset
@@ -239,9 +240,8 @@ servers = set(
     for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x]
                                            for x in args.server))
 
-languages = set(_LANGUAGES[l]
-                for l in itertools.chain.from_iterable(_LANGUAGES.iterkeys(
-                ) if x == 'all' else [x] for x in args.language))
+languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable(
+  six.iterkeys(_LANGUAGES) if x == 'all' else [x] for x in args.language))
 
 docker_images = {}
 # languages for which to build docker images
@@ -267,7 +267,7 @@ if build_jobs:
     jobset.message('FAILED',
                    'Failed to build interop docker images.',
                    do_newline=True)
-    for image in docker_images.itervalues():
+    for image in six.itervalues(docker_images):
       dockerjob.remove_image(image, skip_nonexistent=True)
     sys.exit(1)
 
@@ -306,7 +306,7 @@ try:
 
   if not jobs:
     print('No jobs to run.')
-    for image in docker_images.itervalues():
+    for image in six.itervalues(docker_images):
       dockerjob.remove_image(image, skip_nonexistent=True)
     sys.exit(1)
 
@@ -324,8 +324,8 @@ finally:
     if not job.is_running():
       print('Server "%s" has exited prematurely.' % server)
 
-  dockerjob.finish_jobs([j for j in server_jobs.itervalues()])
+  dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)])
 
-  for image in docker_images.itervalues():
+  for image in six.itervalues(docker_images):
     print('Removing docker image %s' % image)
     dockerjob.remove_image(image)

tools/run_tests/run_tests.py

@@ -54,6 +54,7 @@ import traceback
 import time
 from six.moves import urllib
 import uuid
+import six
 
 import python_utils.jobset as jobset
 import python_utils.report_utils as report_utils
@@ -771,7 +772,7 @@ class CSharpLanguage(object):
         runtime_cmd = ['mono']
 
     specs = []
-    for assembly in tests_by_assembly.iterkeys():
+    for assembly in six.iterkeys(tests_by_assembly):
       assembly_file = 'src/csharp/%s/%s/%s%s' % (assembly,
                                                  assembly_subdir,
                                                  assembly,

tools/run_tests/run_tests_matrix.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -30,6 +30,8 @@
 
 """Run test matrix."""
 
+from __future__ import print_function
+
 import argparse
 import multiprocessing
 import os

tools/run_tests/sanity/check_sources_and_headers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -28,6 +28,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from __future__ import print_function
+
 import json
 import os
 import re

tools/run_tests/sanity/check_test_filtering.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 # Copyright 2016, Google Inc.
 # All rights reserved.
@@ -62,7 +62,7 @@ class TestFilteringTest(unittest.TestCase):
     def _get_changed_files(foo):
       return changed_files
     filter_pull_request_tests._get_changed_files = _get_changed_files
-    print
+    print()
     filtered_jobs = filter_pull_request_tests.filter_tests(all_jobs, "test")
 
     # Make sure sanity tests aren't being filtered out

tools/run_tests/sanity/check_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 # Copyright 2016, Google Inc.
 # All rights reserved.
@@ -29,6 +29,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from __future__ import print_function
+
 import sys
 import yaml
 import os
@@ -48,13 +50,13 @@ try:
     'git rev-parse --abbrev-ref HEAD',
     shell=True)
 except:
-  print 'WARNING: not a git repository'
+  print('WARNING: not a git repository')
   branch_name = None
 
 if branch_name is not None:
   m = re.match(r'^release-([0-9]+)_([0-9]+)$', branch_name)
   if m:
-    print 'RELEASE branch'
+    print('RELEASE branch')
     # version number should align with the branched version
     check_version = lambda version: (
       version.major == int(m.group(1)) and
@@ -78,7 +80,7 @@ settings = build_yaml['settings']
 top_version = Version(settings['version'])
 if not check_version(top_version):
   errors += 1
-  print warning % ('version', top_version)
+  print(warning % ('version', top_version))
 
 for tag, value in settings.iteritems():
   if re.match(r'^[a-z]+_version$', tag):
@@ -86,12 +88,14 @@ for tag, value in settings.iteritems():
     if tag != 'core_version':
       if value.major != top_version.major:
         errors += 1
-        print 'major version mismatch on %s: %d vs %d' % (tag, value.major, top_version.major)
+        print('major version mismatch on %s: %d vs %d' % (tag, value.major,
+                                                          top_version.major))
       if value.minor != top_version.minor:
         errors += 1
-        print 'minor version mismatch on %s: %d vs %d' % (tag, value.minor, top_version.minor)
+        print('minor version mismatch on %s: %d vs %d' % (tag, value.minor,
+                                                          top_version.minor))
     if not check_version(value):
       errors += 1
-      print warning % (tag, value)
+      print(warning % (tag, value))
 
 sys.exit(errors)

tools/run_tests/sanity/core_banned_functions.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 # Copyright 2016, Google Inc.
 # All rights reserved.
@@ -29,6 +29,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from __future__ import print_function
+
 import os
 import sys
 
@@ -60,7 +62,7 @@ for root, dirs, files in os.walk('src/core'):
     for banned, exceptions in BANNED_EXCEPT.items():
       if path in exceptions: continue
       if banned in text:
-        print 'Illegal use of "%s" in %s' % (banned, path)
+        print('Illegal use of "%s" in %s' % (banned, path))
         errors += 1
 
 assert errors == 0

tools/run_tests/start_port_server.py

@@ -39,8 +39,10 @@ The path to this file is called out in test/core/util/port.c, and printed as
 an error message to users.
 """
 
+from __future__ import print_function
+
 import python_utils.start_port_server as start_port_server
 
 start_port_server.start_port_server()
 
-print "Port server started successfully"
+print("Port server started successfully")

tools/run_tests/stress_test/print_summary.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2016, Google Inc.
 # All rights reserved.
 #

tools/run_tests/stress_test/run_on_gke.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2015-2016, Google Inc.
 # All rights reserved.
 #
@@ -27,6 +27,9 @@
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import print_function
+
 import argparse
 import datetime
 import json
@@ -124,23 +127,24 @@ class DockerImage:
     return 'gcr.io/%s/%s' % (project_id, image_name)
 
   def build_image(self):
-    print 'Building docker image: %s (tag: %s)' % (self.image_name,
-                                                   self.tag_name)
+    print('Building docker image: %s (tag: %s)' % (self.image_name,
+                                                   self.tag_name))
     os.environ['INTEROP_IMAGE'] = self.image_name
     os.environ['INTEROP_IMAGE_REPOSITORY_TAG'] = self.tag_name
     os.environ['BASE_NAME'] = self.dockerfile_dir
     os.environ['BUILD_TYPE'] = self.build_type
-    print 'DEBUG: path: ', self.build_script_path
+    print('DEBUG: path: ', self.build_script_path)
     if subprocess.call(args=[self.build_script_path]) != 0:
-      print 'Error in building the Docker image'
+      print('Error in building the Docker image')
       return False
     return True
 
   def push_to_gke_registry(self):
     cmd = ['gcloud', 'docker', 'push', self.tag_name]
-    print 'Pushing %s to the GKE registry..' % self.tag_name
+    print('Pushing %s to the GKE registry..' % self.tag_name)
     if subprocess.call(args=cmd) != 0:
-      print 'Error in pushing the image %s to the GKE registry' % self.tag_name
+      print('Error in pushing the image %s to the GKE registry' %
+            self.tag_name)
       return False
     return True
 
@@ -199,11 +203,11 @@ class Gke:
       cmd = ['kubectl', 'proxy', '--port=%d' % port]
       self.p = subprocess.Popen(args=cmd)
       time.sleep(2)
-      print '\nStarted kubernetes proxy on port: %d' % port
+      print('\nStarted kubernetes proxy on port: %d' % port)
 
     def __del__(self):
       if self.p is not None:
-        print 'Shutting down Kubernetes proxy..'
+        print('Shutting down Kubernetes proxy..')
         self.p.kill()
 
   def __init__(self, project_id, run_id, dataset_id, summary_table_id,
@@ -253,7 +257,7 @@ class Gke:
 
     for pod_name in server_pod_spec.pod_names():
       server_env['POD_NAME'] = pod_name
-      print 'Creating server: %s' % pod_name
+      print('Creating server: %s' % pod_name)
       is_success = kubernetes_api.create_pod_and_service(
           'localhost',
           self.kubernetes_port,
@@ -267,11 +271,11 @@ class Gke:
           True  # Headless = True for server to that GKE creates a DNS record for pod_name
       )
       if not is_success:
-        print 'Error in launching server: %s' % pod_name
+        print('Error in launching server: %s' % pod_name)
         break
 
     if is_success:
-      print 'Successfully created server(s)'
+      print('Successfully created server(s)')
 
     return is_success
 
@@ -301,7 +305,7 @@ class Gke:
 
     for pod_name in client_pod_spec.pod_names():
       client_env['POD_NAME'] = pod_name
-      print 'Creating client: %s' % pod_name
+      print('Creating client: %s' % pod_name)
       is_success = kubernetes_api.create_pod_and_service(
           'localhost',
           self.kubernetes_port,
@@ -316,18 +320,18 @@ class Gke:
       )
 
       if not is_success:
-        print 'Error in launching client %s' % pod_name
+        print('Error in launching client %s' % pod_name)
         break
 
     if is_success:
-      print 'Successfully created all client(s)'
+      print('Successfully created all client(s)')
 
     return is_success
 
   def _delete_pods(self, pod_name_list):
     is_success = True
     for pod_name in pod_name_list:
-      print 'Deleting %s' % pod_name
+      print('Deleting %s' % pod_name)
       is_success = kubernetes_api.delete_pod_and_service(
           'localhost',
           self.kubernetes_port,
@@ -335,11 +339,11 @@ class Gke:
           pod_name)
 
       if not is_success:
-        print 'Error in deleting pod %s' % pod_name
+        print('Error in deleting pod %s' % pod_name)
         break
 
     if is_success:
-      print 'Successfully deleted all pods'
+      print('Successfully deleted all pods')
 
     return is_success
 
@@ -353,7 +357,7 @@ class Gke:
 class Config:
 
   def __init__(self, config_filename, gcp_project_id):
-    print 'Loading configuration...'
+    print('Loading configuration...')
     config_dict = self._load_config(config_filename)
 
     self.global_settings = self._parse_global_settings(config_dict,
@@ -367,7 +371,7 @@ class Config:
     self.client_pod_specs_dict = self._parse_client_pod_specs(
         config_dict, self.docker_images_dict, self.client_templates_dict,
         self.server_pod_specs_dict)
-    print 'Loaded Configuaration.'
+    print('Loaded Configuaration.')
 
   def _parse_global_settings(self, config_dict, gcp_project_id):
     global_settings_dict = config_dict['globalSettings']
@@ -540,8 +544,8 @@ def run_tests(config):
   # run id. This is useful in debugging when looking at records in Biq query)
   run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
   dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id)
-  print 'Run id:', run_id
-  print 'Dataset id:', dataset_id
+  print('Run id:', run_id)
+  print('Dataset id:', dataset_id)
 
   bq_helper = BigQueryHelper(run_id, '', '',
                              config.global_settings.gcp_project_id, dataset_id,
@@ -557,7 +561,7 @@ def run_tests(config):
   is_success = True
 
   try:
-    print 'Launching servers..'
+    print('Launching servers..')
     for name, server_pod_spec in config.server_pod_specs_dict.iteritems():
       if not gke.launch_servers(server_pod_spec):
         is_success = False  # is_success is checked in the 'finally' block
@@ -579,11 +583,12 @@ def run_tests(config):
     start_time = datetime.datetime.now()
     end_time = start_time + datetime.timedelta(
         seconds=config.global_settings.test_duration_secs)
-    print 'Running the test until %s' % end_time.isoformat()
+    print('Running the test until %s' % end_time.isoformat())
 
     while True:
       if datetime.datetime.now() > end_time:
-        print 'Test was run for %d seconds' % config.global_settings.test_duration_secs
+        print('Test was run for %d seconds' %
+              config.global_settings.test_duration_secs)
         break
 
       # Check if either stress server or clients have failed (btw, the bq_helper
@@ -591,11 +596,12 @@ def run_tests(config):
       # have a failure status)
       if bq_helper.check_if_any_tests_failed():
         is_success = False
-        print 'Some tests failed.'
+        print('Some tests failed.')
         break  # Don't 'return' here. We still want to call bq_helper to print qps/summary tables
 
       # Tests running fine. Wait until next poll time to check the status
-      print 'Sleeping for %d seconds..' % config.global_settings.test_poll_interval_secs
+      print('Sleeping for %d seconds..' %
+            config.global_settings.test_poll_interval_secs)
       time.sleep(config.global_settings.test_poll_interval_secs)
 
     # Print BiqQuery tables

tools/run_tests/task_runner.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 # Copyright 2016, Google Inc.
 # All rights reserved.
 #