From 5ea36eb47c8085d64a9efa07010583507168f146 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 17 Nov 2020 16:30:26 -0800 Subject: [PATCH 1/2] Revert "Revert "Resolve warnings and deprecations in several scripts in tools"" --- src/abseil-cpp/gen_build_yaml.py | 2 +- test/cpp/naming/gen_build_yaml.py | 2 +- tools/buildgen/build_cleaner.py | 4 ++-- tools/buildgen/generate_projects.sh | 3 +++ tools/buildgen/mako_renderer.py | 23 ++++++++++++++--------- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/abseil-cpp/gen_build_yaml.py b/src/abseil-cpp/gen_build_yaml.py index 2f32a3e25c7..889c44b4ea9 100755 --- a/src/abseil-cpp/gen_build_yaml.py +++ b/src/abseil-cpp/gen_build_yaml.py @@ -20,7 +20,7 @@ import yaml BUILDS_YAML_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'preprocessed_builds.yaml') with open(BUILDS_YAML_PATH) as f: - builds = yaml.load(f) + builds = yaml.load(f, Loader=yaml.FullLoader) for build in builds: build['build'] = 'private' diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py index 59d12be92a9..eb278572af2 100755 --- a/test/cpp/naming/gen_build_yaml.py +++ b/test/cpp/naming/gen_build_yaml.py @@ -65,7 +65,7 @@ def _resolver_test_cases(resolver_component_data): def main(): resolver_component_data = '' with open('test/cpp/naming/resolver_test_record_groups.yaml') as f: - resolver_component_data = yaml.load(f) + resolver_component_data = yaml.load(f, Loader=yaml.FullLoader) json = { 'resolver_tests_common_zone_name': diff --git a/tools/buildgen/build_cleaner.py b/tools/buildgen/build_cleaner.py index cda6f715123..b044cb7da9b 100755 --- a/tools/buildgen/build_cleaner.py +++ b/tools/buildgen/build_cleaner.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -83,7 +83,7 @@ def cleaned_build_yaml_dict_as_string(indict): if __name__ == '__main__': for filename in sys.argv[1:]: with open(filename) as f: - js = yaml.load(f) + js = yaml.load(f, Loader=yaml.FullLoader) output = cleaned_build_yaml_dict_as_string(js) if TEST: with open(filename) as f: diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh index 81c16153c71..fdbe8dea672 100755 --- a/tools/buildgen/generate_projects.sh +++ b/tools/buildgen/generate_projects.sh @@ -18,6 +18,9 @@ set -e export TEST=${TEST:-false} +# Upgrade Python's YAML library +python3 -m pip install --upgrade --ignore-installed PyYAML + echo "Generating build_autogenerated.yaml from bazel BUILD file" rm -f build_autogenerated.yaml python3 tools/buildgen/extract_metadata_from_bazel_xml.py diff --git a/tools/buildgen/mako_renderer.py b/tools/buildgen/mako_renderer.py index f0202cd5ff9..06ee30ff2a3 100755 --- a/tools/buildgen/mako_renderer.py +++ b/tools/buildgen/mako_renderer.py @@ -15,28 +15,31 @@ """Simple Mako renderer. Just a wrapper around the mako rendering library. - """ import getopt -import imp +import importlib.util import os import pickle import shutil import sys +import yaml from mako.lookup import TemplateLookup from mako.runtime import Context from mako.template import Template + import bunch -import yaml # Imports a plugin -def import_plugin(name): - _, base_ex = os.path.split(name) - base, _ = os.path.splitext(base_ex) - return imp.load_source(base, name) +def import_plugin(path): + module_name = os.path.basename(path).replace('.py', '') + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module def out(msg): @@ -104,7 +107,9 @@ def main(argv): elif opt == '-d': assert not got_preprocessed_input with open(arg, 'r') as dict_file: - bunch.merge_json(json_dict, yaml.load(dict_file.read())) + bunch.merge_json( + json_dict, + yaml.load(dict_file.read(), Loader=yaml.FullLoader)) elif opt == '-p': plugins.append(import_plugin(arg)) elif opt == '-w': @@ -127,7 +132,7 @@ def main(argv): for arg in args: got_input = True with open(arg) as f: - srcs = list(yaml.load_all(f.read())) + srcs = list(yaml.load_all(f.read(), Loader=yaml.FullLoader)) for src in srcs: if isinstance(src, str): assert len(srcs) == 1 From da399bc970667b2dc47eedd69f60e3e23959022d Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 17 Nov 2020 16:34:02 -0800 Subject: [PATCH 2/2] Install yaml in users's directory --- tools/buildgen/generate_projects.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh index fdbe8dea672..9b5c0f50bb2 100755 --- a/tools/buildgen/generate_projects.sh +++ b/tools/buildgen/generate_projects.sh @@ -19,7 +19,7 @@ set -e export TEST=${TEST:-false} # Upgrade Python's YAML library -python3 -m pip install --upgrade --ignore-installed PyYAML +python3 -m pip install --upgrade --ignore-installed PyYAML --user echo "Generating build_autogenerated.yaml from bazel BUILD file" rm -f build_autogenerated.yaml