From 1a718116784e18326f7dd800c4ea3fe773473d19 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 3 Jul 2015 12:07:06 -0700 Subject: [PATCH 1/3] Add parallel execution for generate_projects --- tools/buildgen/generate_projects.py | 75 +++++++++++++++++++++++++++++ tools/buildgen/generate_projects.sh | 6 +++ 2 files changed, 81 insertions(+) create mode 100755 tools/buildgen/generate_projects.py diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py new file mode 100755 index 00000000000..6ea4a05dea5 --- /dev/null +++ b/tools/buildgen/generate_projects.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python2.7 + +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import glob +import os +import sys +import tempfile +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'run_tests')) + +assert sys.argv[1:], 'run generate_projects.sh instead of this directly' + +import jobset + +os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) +json = sys.argv[1:] + +test = {} if 'TEST' in os.environ else None + +plugins = sorted(glob.glob('tools/buildgen/plugins/*.py')) + +jobs = [] +for root, dirs, files in os.walk('templates'): + for f in files: + if os.path.splitext(f)[1] == '.template': + out = '.' + root[len('templates'):] + '/' + os.path.splitext(f)[0] + cmd = ['tools/buildgen/mako_renderer.py'] + for plugin in plugins: + cmd.append('-p') + cmd.append(plugin) + for js in json: + cmd.append('-d') + cmd.append(js) + cmd.append('-o') + if test is None: + cmd.append(out) + else: + test[out] = tempfile.mkstemp() + cmd.append(test[out]) + cmd.append(root + '/' + f) + jobs.append(jobset.JobSpec(cmd, shortname=out)) + +jobset.run(jobs) + +if test is not None: + for s, g in test.iteritems(): + assert(0 == os.system('diff %s %s' % (s, g))) + os.unlink(g) diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh index 53998677465..d4878234b56 100755 --- a/tools/buildgen/generate_projects.sh +++ b/tools/buildgen/generate_projects.sh @@ -45,6 +45,12 @@ fi . tools/buildgen/generate_build_additions.sh +tools/buildgen/generate_projects.py build.json $gen_build_files + +rm $gen_build_files + +exit + global_plugins=`find ./tools/buildgen/plugins -name '*.py' | sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '` From 5cdc81d7090dcdfdd48fe0d6f390daafbf5a328a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 6 Jul 2015 16:36:15 -0700 Subject: [PATCH 2/3] Fix test mode --- tools/buildgen/generate_projects.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 6ea4a05dea5..1964ceb665d 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -62,7 +62,9 @@ for root, dirs, files in os.walk('templates'): if test is None: cmd.append(out) else: - test[out] = tempfile.mkstemp() + tf = tempfile.mkstemp() + test[out] = tf[1] + os.close(tf[0]) cmd.append(test[out]) cmd.append(root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out)) From 89af79fc11048a83a262af33537132870abb3dec Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 9 Jul 2015 16:41:44 -0700 Subject: [PATCH 3/3] Remove dead code --- tools/buildgen/generate_projects.sh | 32 ----------------------------- 1 file changed, 32 deletions(-) diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh index d4878234b56..32fc90fef5d 100755 --- a/tools/buildgen/generate_projects.sh +++ b/tools/buildgen/generate_projects.sh @@ -48,35 +48,3 @@ fi tools/buildgen/generate_projects.py build.json $gen_build_files rm $gen_build_files - -exit - -global_plugins=`find ./tools/buildgen/plugins -name '*.py' | - sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '` - -for dir in . ; do - local_plugins=`find $dir/templates -name '*.py' | - sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '` - - plugins="$global_plugins $local_plugins" - - find -L $dir/templates -type f -and -name *.template | while read file ; do - out=${dir}/${file#$dir/templates/} # strip templates dir prefix - out=${out%.*} # strip template extension - echo "generating file: $out" - json_files="build.json $gen_build_files" - data=`for i in $json_files ; do echo $i ; done | awk ' { printf "-d %s ", $0 } '` - if [ "x$TEST" = "xtrue" ] ; then - actual_out=$out - out=`mktemp /tmp/gentXXXXXX` - fi - mkdir -p `dirname $out` # make sure dest directory exist - $mako_renderer $plugins $data -o $out $file - if [ "x$TEST" = "xtrue" ] ; then - diff -q $out $actual_out - rm $out - fi - done -done - -rm $gen_build_files