Python3-ize tools/buildgen.

* Ran `2to3` on `tools/buildgen` and the `gen_build_yaml.py` files.
* Updated the invocations of python to be explicitly Python 3.
* Changed the loader in `mako_renderer.py` to allow pickle to find the module for pickling `external_version.Version`
* Added a few open-as-binary to placate readers/writers expecting bytes.
* Hand tweaked the templates to remove `iteritems` and fiddle with encodings.

Partially addresses #24359
  
An error occurred
 
.

src/benchmark/gen_build_yaml.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import os
 import sys
 import glob

src/c-ares/gen_build_yaml.py

@@ -145,4 +145,4 @@ try:
 except:
     pass
 
-print yaml.dump(out)
+print(yaml.dump(out))

src/proto/gen_build_yaml.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Generates the appropriate build.json data for all the proto files."""
-from __future__ import print_function
+
 import yaml
 import collections
 import os

src/re2/gen_build_yaml.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import os
 import sys
 import glob

src/upb/gen_build_yaml.py

@@ -16,7 +16,6 @@
 
 # TODO: This should ideally be in upb submodule to avoid hardcoding this here.
 
-from __future__ import print_function
 import re
 import os
 import sys

src/zlib/gen_build_yaml.py

@@ -58,4 +58,4 @@ try:
 except:
     pass
 
-print yaml.dump(out)
+print(yaml.dump(out))

templates/Makefile.template

@@ -127,7 +127,7 @@
 
   # Configurations (as defined under "configs" section in build_handwritten.yaml)
 
-  % for name, args in configs.iteritems():
+  % for name, args in configs.items():
   VALID_CONFIG_${name} = 1
   %  if args.get('compile_the_world', False):
   REQUIRE_CUSTOM_LIBRARIES_${name} = 1
@@ -1020,7 +1020,7 @@
   LIB${lib.name.upper()}_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIB${lib.name.upper()}_SRC))))
 
   % if lib.get('defaults', None):
-  %  for name, value in defaults.get(lib.defaults).iteritems():
+  %  for name, value in defaults.get(lib.defaults).items():
   $(LIB${lib.name.upper()}_OBJS): ${name} += ${value}
   %  endfor
   % endif
@@ -1361,7 +1361,7 @@
   % endif
 
   % if tgt.get('defaults', None):
-  %  for name, value in defaults.get(tgt.defaults).iteritems():
+  %  for name, value in defaults.get(tgt.defaults).items():
   $(${tgt.name.upper()}_OBJS): ${name} += ${value}
   %  endfor
   % endif

templates/grpc.gyp.template

@@ -36,7 +36,7 @@
     },
     'target_defaults': {
       'configurations': {
-        % for name, args in configs.iteritems():
+        % for name, args in configs.items():
         %  if name in ['dbg', 'opt']:
         '${{'dbg':'Debug', 'opt': 'Release'}[name]}': {
           % for arg, prop in [('CPPFLAGS', 'cflags'), ('DEFINES', 'defines')]:

templates/src/objective-c/BoringSSL-GRPC.podspec.template

@@ -244,7 +244,7 @@
       # the correct location in BoringSSL.
       base64 -D <<EOF | gunzip > src/include/openssl/boringssl_prefix_symbols.h
         % for line in compress_boringssl_prefix_header():
-        ${line}
+        ${line.decode('utf-8')}
         % endfor
       EOF
 

templates/src/python/grpcio/grpc_core_dependencies.py.template

@@ -33,7 +33,7 @@
   ]
 
   ASM_SOURCE_FILES = {
-  % for asm, asm_src in asm_srcs.iteritems():
+  % for asm, asm_src in asm_srcs.items():
       '${asm}': [
     % for src in asm_src:
           '${src}',

templates/test/core/end2end/end2end_nosec_tests.cc.template

@@ -1,4 +1,4 @@
 %YAML 1.2
 --- |
   <%namespace file="end2end_defs.include" import="*"/>\
-  ${end2end_selector(k for k, v in core_end2end_tests.iteritems() if not v)}
+  ${end2end_selector(k for k, v in core_end2end_tests.items() if not v)}

templates/tools/run_tests/generated/configs.json.template

@@ -3,7 +3,7 @@
   <%
   import json
   out_configs = []
-  for name, args in configs.iteritems():
+  for name, args in configs.items():
     config_args={}
     config_args['config'] = name
     if args.get('valgrind', None) is not None:

test/core/end2end/gen_build_yaml.py

@@ -33,7 +33,10 @@ def struct(**kwargs):
 # generate_tests.bzl is now the source of truth for end2end tests.
 # The .bzl file is basically a python file and we can "execute" it
 # to get access to the variables it defines.
-execfile('test/core/end2end/generate_tests.bzl')
+exec(
+    compile(
+        open('test/core/end2end/generate_tests.bzl', "rb").read(),
+        'test/core/end2end/generate_tests.bzl', 'exec'))
 
 
 def main():

tools/buildgen/build_cleaner.py

@@ -32,7 +32,8 @@ _ELEM_KEYS = [
 
 
 def repr_ordered_dict(dumper, odict):
-    return dumper.represent_mapping(u'tag:yaml.org,2002:map', odict.items())
+    return dumper.represent_mapping('tag:yaml.org,2002:map',
+                                    list(odict.items()))
 
 
 yaml.add_representer(collections.OrderedDict, repr_ordered_dict)

tools/buildgen/extract_metadata_from_bazel_xml.py

@@ -176,7 +176,8 @@ def _sort_by_build_order(lib_names, lib_dict, deps_key_name, verbose=False):
 
     # all libs that are not in the dictionary are considered external.
     external_deps = list(
-        sorted(filter(lambda lib_name: lib_name not in lib_dict, lib_names)))
+        sorted([lib_name for lib_name in lib_names if lib_name not in lib_dict
+               ]))
     if verbose:
         print('topo_ordering ' + str(lib_names))
         print('    external_deps ' + str(external_deps))
@@ -207,12 +208,12 @@ def _sort_by_build_order(lib_names, lib_dict, deps_key_name, verbose=False):
 def _populate_transitive_deps(bazel_rules):
     """Add 'transitive_deps' field for each of the rules"""
     transitive_deps = {}
-    for rule_name in bazel_rules.iterkeys():
+    for rule_name in bazel_rules.keys():
         transitive_deps[rule_name] = set(bazel_rules[rule_name]['deps'])
 
     while True:
         deps_added = 0
-        for rule_name in bazel_rules.iterkeys():
+        for rule_name in bazel_rules.keys():
             old_deps = transitive_deps[rule_name]
             new_deps = set(old_deps)
             for dep_name in old_deps:
@@ -223,7 +224,7 @@ def _populate_transitive_deps(bazel_rules):
         if deps_added == 0:
             break
 
-    for rule_name, bazel_rule in bazel_rules.iteritems():
+    for rule_name, bazel_rule in bazel_rules.items():
         bazel_rule['transitive_deps'] = list(sorted(transitive_deps[rule_name]))
 
 
@@ -337,7 +338,7 @@ def _expand_intermediate_deps(target_dict, public_dep_names, bazel_rules):
 
 def _generate_build_metadata(build_extra_metadata, bazel_rules):
     """Generate build metadata in build.yaml-like format bazel build metadata and build.yaml-specific "extra metadata"."""
-    lib_names = build_extra_metadata.keys()
+    lib_names = list(build_extra_metadata.keys())
     result = {}
 
     for lib_name in lib_names:
@@ -381,12 +382,13 @@ def _generate_build_metadata(build_extra_metadata, bazel_rules):
 
             # dep names need to be updated as well
             for lib_dict_to_update in result.values():
-                lib_dict_to_update['deps'] = list(
-                    map(lambda dep: to_name if dep == lib_name else dep,
-                        lib_dict_to_update['deps']))
+                lib_dict_to_update['deps'] = list([
+                    to_name if dep == lib_name else dep
+                    for dep in lib_dict_to_update['deps']
+                ])
 
     # make sure deps are listed in reverse topological order (e.g. "grpc gpr" and not "gpr grpc")
-    for lib_dict in result.itervalues():
+    for lib_dict in result.values():
         lib_dict['deps'] = list(
             reversed(_sort_by_build_order(lib_dict['deps'], result, 'deps')))
 
@@ -394,36 +396,35 @@ def _generate_build_metadata(build_extra_metadata, bazel_rules):
 
 
 def _convert_to_build_yaml_like(lib_dict):
-    lib_names = list(
-        filter(
-            lambda lib_name: lib_dict[lib_name].get('_TYPE', 'library') ==
-            'library', lib_dict.keys()))
-    target_names = list(
-        filter(
-            lambda lib_name: lib_dict[lib_name].get('_TYPE', 'library') ==
-            'target', lib_dict.keys()))
-    test_names = list(
-        filter(
-            lambda lib_name: lib_dict[lib_name].get('_TYPE', 'library') ==
-            'test', lib_dict.keys()))
+    lib_names = [
+        lib_name for lib_name in list(lib_dict.keys())
+        if lib_dict[lib_name].get('_TYPE', 'library') == 'library'
+    ]
+    target_names = [
+        lib_name for lib_name in list(lib_dict.keys())
+        if lib_dict[lib_name].get('_TYPE', 'library') == 'target'
+    ]
+    test_names = [
+        lib_name for lib_name in list(lib_dict.keys())
+        if lib_dict[lib_name].get('_TYPE', 'library') == 'test'
+    ]
 
     # list libraries and targets in predefined order
-    lib_list = list(map(lambda lib_name: lib_dict[lib_name], lib_names))
-    target_list = list(map(lambda lib_name: lib_dict[lib_name], target_names))
-    test_list = list(map(lambda lib_name: lib_dict[lib_name], test_names))
+    lib_list = [lib_dict[lib_name] for lib_name in lib_names]
+    target_list = [lib_dict[lib_name] for lib_name in target_names]
+    test_list = [lib_dict[lib_name] for lib_name in test_names]
 
     # get rid of temporary private fields prefixed with "_" and some other useless fields
     for lib in lib_list:
-        for field_to_remove in filter(lambda k: k.startswith('_'), lib.keys()):
+        for field_to_remove in [k for k in lib.keys() if k.startswith('_')]:
             lib.pop(field_to_remove, None)
     for target in target_list:
-        for field_to_remove in filter(lambda k: k.startswith('_'),
-                                      target.keys()):
+        for field_to_remove in [k for k in target.keys() if k.startswith('_')]:
             target.pop(field_to_remove, None)
         target.pop('public_headers',
                    None)  # public headers make no sense for targets
     for test in test_list:
-        for field_to_remove in filter(lambda k: k.startswith('_'), test.keys()):
+        for field_to_remove in [k for k in test.keys() if k.startswith('_')]:
             test.pop(field_to_remove, None)
         test.pop('public_headers',
                  None)  # public headers make no sense for tests
@@ -440,7 +441,7 @@ def _convert_to_build_yaml_like(lib_dict):
 def _extract_cc_tests(bazel_rules):
     """Gets list of cc_test tests from bazel rules"""
     result = []
-    for bazel_rule in bazel_rules.itervalues():
+    for bazel_rule in bazel_rules.values():
         if bazel_rule['class'] == 'cc_test':
             test_name = bazel_rule['name']
             if test_name.startswith('//'):
@@ -453,65 +454,64 @@ def _exclude_unwanted_cc_tests(tests):
     """Filters out bazel tests that we don't want to run with other build systems or we cannot build them reasonably"""
 
     # most qps tests are autogenerated, we are fine without them
-    tests = list(
-        filter(lambda test: not test.startswith('test/cpp/qps:'), tests))
+    tests = [test for test in tests if not test.startswith('test/cpp/qps:')]
 
     # we have trouble with census dependency outside of bazel
-    tests = list(
-        filter(lambda test: not test.startswith('test/cpp/ext/filters/census:'),
-               tests))
-    tests = list(
-        filter(
-            lambda test: not test.startswith(
-                'test/cpp/microbenchmarks:bm_opencensus_plugin'), tests))
+    tests = [
+        test for test in tests
+        if not test.startswith('test/cpp/ext/filters/census:')
+    ]
+    tests = [
+        test for test in tests
+        if not test.startswith('test/cpp/microbenchmarks:bm_opencensus_plugin')
+    ]
 
     # missing opencensus/stats/stats.h
-    tests = list(
-        filter(
-            lambda test: not test.startswith(
-                'test/cpp/end2end:server_load_reporting_end2end_test'), tests))
-    tests = list(
-        filter(
-            lambda test: not test.startswith(
-                'test/cpp/server/load_reporter:lb_load_reporter_test'), tests))
+    tests = [
+        test for test in tests if not test.startswith(
+            'test/cpp/end2end:server_load_reporting_end2end_test')
+    ]
+    tests = [
+        test for test in tests if not test.startswith(
+            'test/cpp/server/load_reporter:lb_load_reporter_test')
+    ]
 
     # The test uses --running_under_bazel cmdline argument
     # To avoid the trouble needing to adjust it, we just skip the test
-    tests = list(
-        filter(
-            lambda test: not test.startswith(
-                'test/cpp/naming:resolver_component_tests_runner_invoker'),
-            tests))
+    tests = [
+        test for test in tests if not test.startswith(
+            'test/cpp/naming:resolver_component_tests_runner_invoker')
+    ]
 
     # the test requires 'client_crash_test_server' to be built
-    tests = list(
-        filter(
-            lambda test: not test.startswith('test/cpp/end2end:time_change_test'
-                                            ), tests))
+    tests = [
+        test for test in tests
+        if not test.startswith('test/cpp/end2end:time_change_test')
+    ]
 
     # the test requires 'client_crash_test_server' to be built
-    tests = list(
-        filter(
-            lambda test: not test.startswith(
-                'test/cpp/end2end:client_crash_test'), tests))
+    tests = [
+        test for test in tests
+        if not test.startswith('test/cpp/end2end:client_crash_test')
+    ]
 
     # the test requires 'server_crash_test_client' to be built
-    tests = list(
-        filter(
-            lambda test: not test.startswith(
-                'test/cpp/end2end:server_crash_test'), tests))
+    tests = [
+        test for test in tests
+        if not test.startswith('test/cpp/end2end:server_crash_test')
+    ]
 
     # test never existed under build.yaml and it fails -> skip it
-    tests = list(
-        filter(
-            lambda test: not test.startswith(
-                'test/core/tsi:ssl_session_cache_test'), tests))
+    tests = [
+        test for test in tests
+        if not test.startswith('test/core/tsi:ssl_session_cache_test')
+    ]
 
     # the binary of this test does not get built with cmake
-    tests = list(
-        filter(
-            lambda test: not test.startswith(
-                'test/cpp/util:channelz_sampler_test'), tests))
+    tests = [
+        test for test in tests
+        if not test.startswith('test/cpp/util:channelz_sampler_test')
+    ]
 
     return tests
 
@@ -594,14 +594,14 @@ def _generate_build_extra_metadata_for_tests(tests, bazel_rules):
 
     # detect duplicate test names
     tests_by_simple_name = {}
-    for test_name, test_dict in test_metadata.iteritems():
+    for test_name, test_dict in test_metadata.items():
         simple_test_name = test_dict['_RENAME']
         if not simple_test_name in tests_by_simple_name:
             tests_by_simple_name[simple_test_name] = []
         tests_by_simple_name[simple_test_name].append(test_name)
 
     # choose alternative names for tests with a name collision
-    for collision_list in tests_by_simple_name.itervalues():
+    for collision_list in tests_by_simple_name.values():
         if len(collision_list) > 1:
             for test_name in collision_list:
                 long_name = test_name.replace('/', '_').replace(':', '_')

tools/buildgen/generate_build_additions.sh

@@ -33,6 +33,6 @@ gen_build_files=""
 for gen_build_yaml in $gen_build_yaml_dirs
 do
   output_file=`mktemp /tmp/genXXXXXX`
-  python $gen_build_yaml/gen_build_yaml.py > $output_file
+  python3 $gen_build_yaml/gen_build_yaml.py > $output_file
   gen_build_files="$gen_build_files $output_file"
 done

tools/buildgen/generate_projects.py

@@ -93,7 +93,7 @@ jobset.run(pre_jobs, maxjobs=args.jobs)
 jobset.run(jobs, maxjobs=args.jobs)
 
 if test is not None:
-    for s, g in test.iteritems():
+    for s, g in test.items():
         if os.path.isfile(g):
             assert 0 == os.system('diff %s %s' % (s, g)), s
             os.unlink(g)

tools/buildgen/generate_projects.sh

@@ -20,7 +20,7 @@ export TEST=${TEST:-false}
 
 echo "Generating build_autogenerated.yaml from bazel BUILD file"
 rm -f build_autogenerated.yaml
-python tools/buildgen/extract_metadata_from_bazel_xml.py
+python3 tools/buildgen/extract_metadata_from_bazel_xml.py
 
 cd `dirname $0`/../..
 mako_renderer=tools/buildgen/mako_renderer.py
@@ -35,6 +35,6 @@ TEST=true tools/buildgen/build_cleaner.py build_autogenerated.yaml
 # Instead of generating from a single build.yaml, we've split it into
 # - build_handwritten.yaml: manually written metadata
 # - build_autogenerated.yaml: generated from bazel BUILD file
-python tools/buildgen/generate_projects.py build_handwritten.yaml build_autogenerated.yaml $gen_build_files $*
+python3 tools/buildgen/generate_projects.py build_handwritten.yaml build_autogenerated.yaml $gen_build_files $*
 
 rm $gen_build_files

tools/buildgen/mako_renderer.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 # Copyright 2015 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -21,7 +21,7 @@ Just a wrapper around the mako rendering library.
 import getopt
 import imp
 import os
-import cPickle as pickle
+import pickle
 import shutil
 import sys
 
@@ -36,16 +36,11 @@ import yaml
 def import_plugin(name):
     _, base_ex = os.path.split(name)
     base, _ = os.path.splitext(base_ex)
-
-    with open(name, 'r') as plugin_file:
-        plugin_code = plugin_file.read()
-    plugin_module = imp.new_module(base)
-    exec plugin_code in plugin_module.__dict__
-    return plugin_module
+    return imp.load_source(base, name)
 
 
 def out(msg):
-    print >> sys.stderr, msg
+    print(msg, file=sys.stderr)
 
 
 def showhelp():
@@ -103,7 +98,7 @@ def main(argv):
                 0,
                 os.path.abspath(
                     os.path.join(os.path.dirname(sys.argv[0]), 'plugins')))
-            with open(arg, 'r') as dict_file:
+            with open(arg, 'rb') as dict_file:
                 dictionary = pickle.load(dict_file)
             got_preprocessed_input = True
         elif opt == '-d':
@@ -125,7 +120,7 @@ def main(argv):
             dictionary[k] = bunch.to_bunch(v)
 
     if preprocessed_output:
-        with open(preprocessed_output, 'w') as dict_file:
+        with open(preprocessed_output, 'wb') as dict_file:
             pickle.dump(dictionary, dict_file)
 
     cleared_dir = False
@@ -134,7 +129,7 @@ def main(argv):
         with open(arg) as f:
             srcs = list(yaml.load_all(f.read()))
         for src in srcs:
-            if isinstance(src, basestring):
+            if isinstance(src, str):
                 assert len(srcs) == 1
                 template = Template(src,
                                     filename=arg,

tools/buildgen/plugins/expand_filegroups.py

@@ -62,7 +62,7 @@ def mako_plugin(dictionary):
         for lst in FILEGROUP_LISTS:
             fg[lst] = fg.get(lst, [])
             fg['own_%s' % lst] = list(fg[lst])
-        for attr, val in FILEGROUP_DEFAULTS.iteritems():
+        for attr, val in FILEGROUP_DEFAULTS.items():
             if attr not in fg:
                 fg[attr] = val
 
@@ -113,7 +113,7 @@ def mako_plugin(dictionary):
         thing['used_by'] = []
     thing_deps = lambda t: t.get('uses', []) + t.get('filegroups', []) + t.get(
         'deps', [])
-    for thing in things.itervalues():
+    for thing in things.values():
         done = set()
         todo = thing_deps(thing)
         while todo:
@@ -125,7 +125,7 @@ def mako_plugin(dictionary):
             done.add(cur)
 
     # the above expansion can introduce duplicate filenames: contract them here
-    for fg in filegroups.itervalues():
+    for fg in filegroups.values():
         for lst in FILEGROUP_LISTS:
             fg[lst] = uniquify(fg.get(lst, []))
 

tools/buildgen/plugins/generate_vsprojects.py

@@ -61,7 +61,7 @@ def mako_plugin(dictionary):
             name = target['name']
             guid = re.sub('(........)(....)(....)(....)(.*)',
                           r'{\1-\2-\3-\4-\5}',
-                          hashlib.md5(name).hexdigest())
+                          hashlib.md5(name.encode('utf-8')).hexdigest())
             target['vs_project_guid'] = guid.upper()
     # Exclude projects without a visual project guid, such as the tests.
     projects = [

tools/buildgen/plugins/list_api.py

@@ -64,4 +64,5 @@ def mako_plugin(dictionary):
 
 
 if __name__ == '__main__':
-    print yaml.dump([api for api in list_c_apis(headers_under('include/grpc'))])
+    print(yaml.dump([api for api in list_c_apis(headers_under('include/grpc'))
+                    ]))