grpc/tools/buildgen/generate_projects.py

# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import glob
import yaml
import pickle
import os
import shutil
import sys
import tempfile
import multiprocessing
from typing import Union, Dict, List

import _utils

PROJECT_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..",
                            "..")
os.chdir(PROJECT_ROOT)
# TODO(lidiz) find a better way for plugins to reference each other
sys.path.append(os.path.join(PROJECT_ROOT, 'tools', 'buildgen', 'plugins'))

# from tools.run_tests.python_utils import jobset
jobset = _utils.import_python_module(
    os.path.join(PROJECT_ROOT, 'tools', 'run_tests', 'python_utils',
                 'jobset.py'))

PREPROCESSED_BUILD = '.preprocessed_build'
test = {} if os.environ.get('TEST', 'false') == 'true' else None

assert sys.argv[1:], 'run generate_projects.sh instead of this directly'
parser = argparse.ArgumentParser()
parser.add_argument('build_files',
                    nargs='+',
                    default=[],
                    help="build files describing build specs")
parser.add_argument('--templates',
                    nargs='+',
                    default=[],
                    help="mako template files to render")
parser.add_argument('--output_merged',
                    '-m',
                    default='',
                    type=str,
                    help="merge intermediate results to a file")
parser.add_argument('--jobs',
                    '-j',
                    default=multiprocessing.cpu_count(),
                    type=int,
                    help="maximum parallel jobs")
parser.add_argument('--base',
                    default='.',
                    type=str,
                    help="base path for generated files")
args = parser.parse_args()


def preprocess_build_files() -> _utils.Bunch:
    """Merges build yaml into a one dictionary then pass it to plugins."""
    build_spec = dict()
    for build_file in args.build_files:
        with open(build_file, 'r') as f:
            _utils.merge_json(build_spec,
                              yaml.load(f.read(), Loader=yaml.FullLoader))
    # Executes plugins. Plugins update the build spec in-place.
    for py_file in sorted(glob.glob('tools/buildgen/plugins/*.py')):
        plugin = _utils.import_python_module(py_file)
        plugin.mako_plugin(build_spec)
    if args.output_merged:
        with open(args.output_merged, 'w') as f:
            f.write(yaml.dump(build_spec))
    # Makes build_spec sort of immutable and dot-accessible
    return _utils.to_bunch(build_spec)


def generate_template_render_jobs(templates: List[str]) -> List[jobset.JobSpec]:
    """Generate JobSpecs for each one of the template rendering work."""
    jobs = []
    base_cmd = [sys.executable, 'tools/buildgen/_mako_renderer.py']
    for template in sorted(templates, reverse=True):
        root, f = os.path.split(template)
        if os.path.splitext(f)[1] == '.template':
            out_dir = args.base + root[len('templates'):]
            out = os.path.join(out_dir, os.path.splitext(f)[0])
            if not os.path.exists(out_dir):
                os.makedirs(out_dir)
            cmd = base_cmd[:]
            cmd.append('-P')
            cmd.append(PREPROCESSED_BUILD)
            cmd.append('-o')
            if test is None:
                cmd.append(out)
            else:
                tf = tempfile.mkstemp()
                test[out] = tf[1]
                os.close(tf[0])
                cmd.append(test[out])
            cmd.append(args.base + '/' + root + '/' + f)
            jobs.append(jobset.JobSpec(cmd, shortname=out,
                                       timeout_seconds=None))
    return jobs


def main() -> None:
    templates = args.templates
    if not templates:
        for root, _, files in os.walk('templates'):
            for f in files:
                templates.append(os.path.join(root, f))

    build_spec = preprocess_build_files()
    with open(PREPROCESSED_BUILD, 'wb') as f:
        pickle.dump(build_spec, f)

    err_cnt, _ = jobset.run(generate_template_render_jobs(templates),
                            maxjobs=args.jobs)
    if err_cnt != 0:
        print(f'ERROR: {err_cnt} error(s) found while generating projects.',
              file=sys.stderr)
        sys.exit(1)

    if test is not None:
        for s, g in test.items():
            if os.path.isfile(g):
                assert 0 == os.system('diff %s %s' % (s, g)), s
                os.unlink(g)
            else:
                assert 0 == os.system('diff -r %s %s' % (s, g)), s
                shutil.rmtree(g, ignore_errors=True)


if __name__ == "__main__":
    main()