Merge pull request #24361 from veblush/py2to3-sanity

Porting sanity & distrib script to Python3

templates/tools/dockerfile/test/sanity/Dockerfile.template

@@ -31,7 +31,7 @@
         curl ${"\\"}
         shellcheck
   RUN python2 -m pip install simplejson mako virtualenv==16.7.9 lxml
-  RUN python3 -m pip install simplejson mako virtualenv==16.7.9 lxml
+  RUN python3 -m pip install simplejson mako virtualenv==16.7.9 lxml six
 
   # Add buster-backports for more recent clang packages
   RUN echo "deb http://deb.debian.org/debian buster-backports main" | tee /etc/apt/sources.list.d/buster-backports.list

tools/distrib/check_copyright.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 
 # Copyright 2015 gRPC authors.
 #
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import argparse
 import datetime
 import os
@@ -107,7 +106,7 @@ RE_LICENSE = dict(
     (k, r'\n'.join(LICENSE_PREFIX[k] +
                    (RE_YEAR if re.search(RE_YEAR, line) else re.escape(line))
                    for line in LICENSE_NOTICE))
-    for k, v in LICENSE_PREFIX.iteritems())
+    for k, v in LICENSE_PREFIX.items())
 
 if args.precommit:
     FILE_LIST_COMMAND = 'git status -z | grep -Poz \'(?<=^[MARC][MARCD ] )[^\s]+\''
@@ -143,7 +142,7 @@ ok = True
 filename_list = []
 try:
     filename_list = subprocess.check_output(FILE_LIST_COMMAND,
-                                            shell=True).splitlines()
+                                            shell=True).decode().splitlines()
 except subprocess.CalledProcessError:
     sys.exit(0)
 

tools/distrib/check_include_guards.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 
 # Copyright 2016 gRPC authors.
 #
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import argparse
 import os
 import os.path
@@ -177,7 +176,7 @@ ok = True
 filename_list = []
 try:
     filename_list = subprocess.check_output(FILE_LIST_COMMAND,
-                                            shell=True).splitlines()
+                                            shell=True).decode().splitlines()
     # Filter out non-existent files (ie, file removed or renamed)
     filename_list = (f for f in filename_list if os.path.isfile(f))
 except subprocess.CalledProcessError:

tools/dockerfile/test/sanity/Dockerfile

@@ -79,7 +79,7 @@ RUN apt-get update && apt-get install -y \
       curl \
       shellcheck
 RUN python2 -m pip install simplejson mako virtualenv==16.7.9 lxml
-RUN python3 -m pip install simplejson mako virtualenv==16.7.9 lxml
+RUN python3 -m pip install simplejson mako virtualenv==16.7.9 lxml six
 
 # Add buster-backports for more recent clang packages
 RUN echo "deb http://deb.debian.org/debian buster-backports main" | tee /etc/apt/sources.list.d/buster-backports.list

tools/run_tests/sanity/check_bazel_workspace.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 gRPC authors.
 #
@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import ast
 import os
 import re
@@ -27,8 +25,8 @@ os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..'))
 git_hash_pattern = re.compile('[0-9a-f]{40}')
 
 # Parse git hashes from submodules
-git_submodules = subprocess.check_output('git submodule',
+git_submodules = subprocess.check_output(
-                                         shell=True).strip().split('\n')
+    'git submodule', shell=True).decode().strip().split('\n')
 git_submodule_hashes = {
     re.search(git_hash_pattern, s).group() for s in git_submodules
 }
@@ -148,17 +146,14 @@ build_rules = {
     'git_repository': lambda **args: eval_state.git_repository(**args),
     'grpc_python_deps': lambda: None,
 }
-exec(bazel_file) in build_rules
+exec((bazel_file), build_rules)
 for name in _GRPC_DEP_NAMES:
-    assert name in names_and_urls.keys()
+    assert name in list(names_and_urls.keys())
-if len(_GRPC_DEP_NAMES) != len(names_and_urls.keys()):
+assert len(_GRPC_DEP_NAMES) == len(list(names_and_urls.keys()))
-    assert False, "Diff: " + (str(set(_GRPC_DEP_NAMES) - set(names_and_urls)) +
-                              "," +
-                              str(set(names_and_urls) - set(_GRPC_DEP_NAMES)))
 
 # There are some "bazel-only" deps that are exceptions to this sanity check,
 # we don't require that there is a corresponding git module for these.
-names_without_bazel_only_deps = names_and_urls.keys()
+names_without_bazel_only_deps = list(names_and_urls.keys())
 for dep_name in _GRPC_BAZEL_ONLY_DEPS:
     names_without_bazel_only_deps.remove(dep_name)
 archive_urls = [names_and_urls[name] for name in names_without_bazel_only_deps]
@@ -195,7 +190,7 @@ for name in _GRPC_DEP_NAMES:
         'git_repository': lambda **args: state.git_repository(**args),
         'grpc_python_deps': lambda *args, **kwargs: None,
     }
-    exec(bazel_file) in rules
+    exec((bazel_file), rules)
-    assert name not in names_and_urls_with_overridden_name.keys()
+    assert name not in list(names_and_urls_with_overridden_name.keys())
 
 sys.exit(0)

tools/run_tests/sanity/check_deprecated_grpc++.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2018 gRPC authors.
 #
@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import os
 import sys
 

tools/run_tests/sanity/check_port_platform.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2017 gRPC authors.
 #
@@ -64,6 +64,6 @@ all_bad_files += check_port_platform_inclusion(os.path.join('include', 'grpc'))
 
 if len(all_bad_files) > 0:
     for f in all_bad_files:
-        print(('port_platform.h is not the first included header or there '
+        print((('port_platform.h is not the first included header or there '
-               'is not a blank line following its inclusion in %s') % f)
+                'is not a blank line following its inclusion in %s') % f))
     sys.exit(1)

tools/run_tests/sanity/check_test_filtering.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 gRPC authors.
 #
@@ -67,8 +67,8 @@ class TestFilteringTest(unittest.TestCase):
         filtered_jobs = [
             job for job in filtered_jobs if "sanity" not in job.labels
         ]
-        self.assertEquals(sanity_tests_in_all_jobs,
+        self.assertEqual(sanity_tests_in_all_jobs,
-                          sanity_tests_in_filtered_jobs)
+                         sanity_tests_in_filtered_jobs)
 
         for label in labels:
             for job in filtered_jobs:
@@ -79,8 +79,8 @@ class TestFilteringTest(unittest.TestCase):
             for job in all_jobs:
                 if (label in job.labels):
                     jobs_matching_labels += 1
-        self.assertEquals(len(filtered_jobs),
+        self.assertEqual(len(filtered_jobs),
-                          len(all_jobs) - jobs_matching_labels)
+                         len(all_jobs) - jobs_matching_labels)
 
     def test_individual_language_filters(self):
         # Changing unlisted file should trigger all languages
@@ -152,7 +152,7 @@ class TestFilteringTest(unittest.TestCase):
             'src/core/foo.bar', 'some_file_not_on_the_white_list', 'BUILD',
             'etc/roots.pem', 'Makefile', 'tools/foo'
         ]
-        for key in whitelist.keys():
+        for key in list(whitelist.keys()):
             for file_name in files_that_should_trigger_all_tests:
                 self.assertFalse(re.match(key, file_name))
 

tools/run_tests/sanity/check_tracer_sanity.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2017 gRPC authors.
 #
@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import os
 import sys
 import re

tools/run_tests/sanity/check_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 gRPC authors.
 #
@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import sys
 import yaml
 import os
@@ -66,7 +64,7 @@ if not check_version(top_version):
     errors += 1
     print(warning % ('version', top_version))
 
-for tag, value in settings.iteritems():
+for tag, value in settings.items():
     if re.match(r'^[a-z]+_version$', tag):
         value = Version(value)
         if tag != 'core_version':

tools/run_tests/sanity/core_banned_functions.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 gRPC authors.
 #
@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import os
 import sys