Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #5395 from ctiller/faster-generation

Browse Source 
Optimize generate_projects
pull/5404/head
Nicolas Noble 9 years ago
parent b0ec2baa15 560c9017c4
commit 7466d3397b
6 changed files with 160 additions and 133 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      .gitignore
  2. 10
      config.m4
  3. 192
      package.json
  4. 19
      package.xml
  5. 31
      tools/buildgen/generate_projects.py
  6. 40
      tools/buildgen/mako_renderer.py

1
.gitignore vendored
Unescape View File

@ -35,6 +35,7 @@ coverage
# cache for run_tests.py
.run_tests_cache
.preprocessed_build
# emacs temp files
*~

10
config.m4
Unescape View File

@ -48,11 +48,9 @@ if test "$PHP_GRPC" != "no"; then
src/core/support/env_linux.c \
src/core/support/env_posix.c \
src/core/support/env_win32.c \
src/core/support/file.c \
src/core/support/file_posix.c \
src/core/support/file_win32.c \
src/core/support/histogram.c \
src/core/support/host_port.c \
src/core/support/load_file.c \
src/core/support/log.c \
src/core/support/log_android.c \
src/core/support/log_linux.c \
@ -78,6 +76,8 @@ if test "$PHP_GRPC" != "no"; then
src/core/support/time_precise.c \
src/core/support/time_win32.c \
src/core/support/tls_pthread.c \
src/core/support/tmpfile_posix.c \
src/core/support/tmpfile_win32.c \
src/core/support/wrap_memcpy.c \
src/core/census/grpc_context.c \
src/core/census/grpc_filter.c \
@ -109,7 +109,7 @@ if test "$PHP_GRPC" != "no"; then
src/core/client_config/subchannel_factory.c \
src/core/client_config/subchannel_index.c \
src/core/client_config/uri_parser.c \
src/core/compression/algorithm.c \
src/core/compression/compression_algorithm.c \
src/core/compression/message_compress.c \
src/core/debug/trace.c \
src/core/httpcli/format_request.c \
@ -210,7 +210,7 @@ if test "$PHP_GRPC" != "no"; then
src/core/transport/transport.c \
src/core/transport/transport_op_string.c \
src/core/httpcli/httpcli_security_connector.c \
src/core/security/base64.c \
src/core/security/b64.c \
src/core/security/client_auth_filter.c \
src/core/security/credentials.c \
src/core/security/credentials_metadata.c \

192
package.json
Unescape View File

@ -421,102 +421,6 @@
"third_party/zlib/trees.c",
"third_party/zlib/uncompr.c",
"third_party/zlib/zutil.c",
"include/grpc/support/alloc.h",
"include/grpc/support/atm.h",
"include/grpc/support/atm_gcc_atomic.h",
"include/grpc/support/atm_gcc_sync.h",
"include/grpc/support/atm_win32.h",
"include/grpc/support/avl.h",
"include/grpc/support/cmdline.h",
"include/grpc/support/cpu.h",
"include/grpc/support/histogram.h",
"include/grpc/support/host_port.h",
"include/grpc/support/log.h",
"include/grpc/support/log_win32.h",
"include/grpc/support/port_platform.h",
"include/grpc/support/slice.h",
"include/grpc/support/slice_buffer.h",
"include/grpc/support/string_util.h",
"include/grpc/support/subprocess.h",
"include/grpc/support/sync.h",
"include/grpc/support/sync_generic.h",
"include/grpc/support/sync_posix.h",
"include/grpc/support/sync_win32.h",
"include/grpc/support/thd.h",
"include/grpc/support/time.h",
"include/grpc/support/tls.h",
"include/grpc/support/tls_gcc.h",
"include/grpc/support/tls_msvc.h",
"include/grpc/support/tls_pthread.h",
"include/grpc/support/useful.h",
"include/grpc/impl/codegen/alloc.h",
"include/grpc/impl/codegen/atm.h",
"include/grpc/impl/codegen/atm_gcc_atomic.h",
"include/grpc/impl/codegen/atm_gcc_sync.h",
"include/grpc/impl/codegen/atm_win32.h",
"include/grpc/impl/codegen/log.h",
"include/grpc/impl/codegen/port_platform.h",
"include/grpc/impl/codegen/slice.h",
"include/grpc/impl/codegen/slice_buffer.h",
"include/grpc/impl/codegen/sync.h",
"include/grpc/impl/codegen/sync_generic.h",
"include/grpc/impl/codegen/sync_posix.h",
"include/grpc/impl/codegen/sync_win32.h",
"include/grpc/impl/codegen/time.h",
"src/core/profiling/timers.h",
"src/core/support/block_annotate.h",
"src/core/support/env.h",
"src/core/support/load_file.h",
"src/core/support/murmur_hash.h",
"src/core/support/stack_lockfree.h",
"src/core/support/string.h",
"src/core/support/string_win32.h",
"src/core/support/thd_internal.h",
"src/core/support/time_precise.h",
"src/core/support/tmpfile.h",
"src/core/profiling/basic_timers.c",
"src/core/profiling/stap_timers.c",
"src/core/support/alloc.c",
"src/core/support/avl.c",
"src/core/support/cmdline.c",
"src/core/support/cpu_iphone.c",
"src/core/support/cpu_linux.c",
"src/core/support/cpu_posix.c",
"src/core/support/cpu_windows.c",
"src/core/support/env_linux.c",
"src/core/support/env_posix.c",
"src/core/support/env_win32.c",
"src/core/support/histogram.c",
"src/core/support/host_port.c",
"src/core/support/load_file.c",
"src/core/support/log.c",
"src/core/support/log_android.c",
"src/core/support/log_linux.c",
"src/core/support/log_posix.c",
"src/core/support/log_win32.c",
"src/core/support/murmur_hash.c",
"src/core/support/slice.c",
"src/core/support/slice_buffer.c",
"src/core/support/stack_lockfree.c",
"src/core/support/string.c",
"src/core/support/string_posix.c",
"src/core/support/string_win32.c",
"src/core/support/subprocess_posix.c",
"src/core/support/subprocess_windows.c",
"src/core/support/sync.c",
"src/core/support/sync_posix.c",
"src/core/support/sync_win32.c",
"src/core/support/thd.c",
"src/core/support/thd_posix.c",
"src/core/support/thd_win32.c",
"src/core/support/time.c",
"src/core/support/time_posix.c",
"src/core/support/time_precise.c",
"src/core/support/time_win32.c",
"src/core/support/tls_pthread.c",
"src/core/support/tmpfile_posix.c",
"src/core/support/tmpfile_win32.c",
"src/core/support/wrap_memcpy.c",
"third_party/boringssl/crypto/aes/internal.h",
"third_party/boringssl/crypto/asn1/asn1_locl.h",
"third_party/boringssl/crypto/bio/internal.h",
@ -918,6 +822,102 @@
"third_party/boringssl/ssl/t1_enc.c",
"third_party/boringssl/ssl/t1_lib.c",
"third_party/boringssl/ssl/tls_record.c",
"include/grpc/support/alloc.h",
"include/grpc/support/atm.h",
"include/grpc/support/atm_gcc_atomic.h",
"include/grpc/support/atm_gcc_sync.h",
"include/grpc/support/atm_win32.h",
"include/grpc/support/avl.h",
"include/grpc/support/cmdline.h",
"include/grpc/support/cpu.h",
"include/grpc/support/histogram.h",
"include/grpc/support/host_port.h",
"include/grpc/support/log.h",
"include/grpc/support/log_win32.h",
"include/grpc/support/port_platform.h",
"include/grpc/support/slice.h",
"include/grpc/support/slice_buffer.h",
"include/grpc/support/string_util.h",
"include/grpc/support/subprocess.h",
"include/grpc/support/sync.h",
"include/grpc/support/sync_generic.h",
"include/grpc/support/sync_posix.h",
"include/grpc/support/sync_win32.h",
"include/grpc/support/thd.h",
"include/grpc/support/time.h",
"include/grpc/support/tls.h",
"include/grpc/support/tls_gcc.h",
"include/grpc/support/tls_msvc.h",
"include/grpc/support/tls_pthread.h",
"include/grpc/support/useful.h",
"include/grpc/impl/codegen/alloc.h",
"include/grpc/impl/codegen/atm.h",
"include/grpc/impl/codegen/atm_gcc_atomic.h",
"include/grpc/impl/codegen/atm_gcc_sync.h",
"include/grpc/impl/codegen/atm_win32.h",
"include/grpc/impl/codegen/log.h",
"include/grpc/impl/codegen/port_platform.h",
"include/grpc/impl/codegen/slice.h",
"include/grpc/impl/codegen/slice_buffer.h",
"include/grpc/impl/codegen/sync.h",
"include/grpc/impl/codegen/sync_generic.h",
"include/grpc/impl/codegen/sync_posix.h",
"include/grpc/impl/codegen/sync_win32.h",
"include/grpc/impl/codegen/time.h",
"src/core/profiling/timers.h",
"src/core/support/block_annotate.h",
"src/core/support/env.h",
"src/core/support/load_file.h",
"src/core/support/murmur_hash.h",
"src/core/support/stack_lockfree.h",
"src/core/support/string.h",
"src/core/support/string_win32.h",
"src/core/support/thd_internal.h",
"src/core/support/time_precise.h",
"src/core/support/tmpfile.h",
"src/core/profiling/basic_timers.c",
"src/core/profiling/stap_timers.c",
"src/core/support/alloc.c",
"src/core/support/avl.c",
"src/core/support/cmdline.c",
"src/core/support/cpu_iphone.c",
"src/core/support/cpu_linux.c",
"src/core/support/cpu_posix.c",
"src/core/support/cpu_windows.c",
"src/core/support/env_linux.c",
"src/core/support/env_posix.c",
"src/core/support/env_win32.c",
"src/core/support/histogram.c",
"src/core/support/host_port.c",
"src/core/support/load_file.c",
"src/core/support/log.c",
"src/core/support/log_android.c",
"src/core/support/log_linux.c",
"src/core/support/log_posix.c",
"src/core/support/log_win32.c",
"src/core/support/murmur_hash.c",
"src/core/support/slice.c",
"src/core/support/slice_buffer.c",
"src/core/support/stack_lockfree.c",
"src/core/support/string.c",
"src/core/support/string_posix.c",
"src/core/support/string_win32.c",
"src/core/support/subprocess_posix.c",
"src/core/support/subprocess_windows.c",
"src/core/support/sync.c",
"src/core/support/sync_posix.c",
"src/core/support/sync_win32.c",
"src/core/support/thd.c",
"src/core/support/thd_posix.c",
"src/core/support/thd_win32.c",
"src/core/support/time.c",
"src/core/support/time_posix.c",
"src/core/support/time_precise.c",
"src/core/support/time_win32.c",
"src/core/support/tls_pthread.c",
"src/core/support/tmpfile_posix.c",
"src/core/support/tmpfile_win32.c",
"src/core/support/wrap_memcpy.c",
"binding.gyp"
],
"main": "src/node/index.js",

19
package.xml
Unescape View File

@ -10,7 +10,7 @@
<email>grpc-packages@google.com</email>
<active>yes</active>
</lead>
<date>2016-02-23</date>
<date>2016-02-24</date>
<time>16:06:07</time>
<version>
<release>0.8.0</release>
@ -95,13 +95,14 @@
<file baseinstalldir="/" name="src/core/profiling/timers.h" role="src" />
<file baseinstalldir="/" name="src/core/support/block_annotate.h" role="src" />
<file baseinstalldir="/" name="src/core/support/env.h" role="src" />
<file baseinstalldir="/" name="src/core/support/file.h" role="src" />
<file baseinstalldir="/" name="src/core/support/load_file.h" role="src" />
<file baseinstalldir="/" name="src/core/support/murmur_hash.h" role="src" />
<file baseinstalldir="/" name="src/core/support/stack_lockfree.h" role="src" />
<file baseinstalldir="/" name="src/core/support/string.h" role="src" />
<file baseinstalldir="/" name="src/core/support/string_win32.h" role="src" />
<file baseinstalldir="/" name="src/core/support/thd_internal.h" role="src" />
<file baseinstalldir="/" name="src/core/support/time_precise.h" role="src" />
<file baseinstalldir="/" name="src/core/support/tmpfile.h" role="src" />
<file baseinstalldir="/" name="src/core/profiling/basic_timers.c" role="src" />
<file baseinstalldir="/" name="src/core/profiling/stap_timers.c" role="src" />
<file baseinstalldir="/" name="src/core/support/alloc.c" role="src" />
@ -114,11 +115,9 @@
<file baseinstalldir="/" name="src/core/support/env_linux.c" role="src" />
<file baseinstalldir="/" name="src/core/support/env_posix.c" role="src" />
<file baseinstalldir="/" name="src/core/support/env_win32.c" role="src" />
<file baseinstalldir="/" name="src/core/support/file.c" role="src" />
<file baseinstalldir="/" name="src/core/support/file_posix.c" role="src" />
<file baseinstalldir="/" name="src/core/support/file_win32.c" role="src" />
<file baseinstalldir="/" name="src/core/support/histogram.c" role="src" />
<file baseinstalldir="/" name="src/core/support/host_port.c" role="src" />
<file baseinstalldir="/" name="src/core/support/load_file.c" role="src" />
<file baseinstalldir="/" name="src/core/support/log.c" role="src" />
<file baseinstalldir="/" name="src/core/support/log_android.c" role="src" />
<file baseinstalldir="/" name="src/core/support/log_linux.c" role="src" />
@ -144,6 +143,8 @@
<file baseinstalldir="/" name="src/core/support/time_precise.c" role="src" />
<file baseinstalldir="/" name="src/core/support/time_win32.c" role="src" />
<file baseinstalldir="/" name="src/core/support/tls_pthread.c" role="src" />
<file baseinstalldir="/" name="src/core/support/tmpfile_posix.c" role="src" />
<file baseinstalldir="/" name="src/core/support/tmpfile_win32.c" role="src" />
<file baseinstalldir="/" name="src/core/support/wrap_memcpy.c" role="src" />
<file baseinstalldir="/" name="include/grpc/grpc_security.h" role="src" />
<file baseinstalldir="/" name="include/grpc/impl/codegen/byte_buffer.h" role="src" />
@ -274,7 +275,7 @@
<file baseinstalldir="/" name="src/core/transport/transport.h" role="src" />
<file baseinstalldir="/" name="src/core/transport/transport_impl.h" role="src" />
<file baseinstalldir="/" name="src/core/security/auth_filters.h" role="src" />
<file baseinstalldir="/" name="src/core/security/base64.h" role="src" />
<file baseinstalldir="/" name="src/core/security/b64.h" role="src" />
<file baseinstalldir="/" name="src/core/security/credentials.h" role="src" />
<file baseinstalldir="/" name="src/core/security/handshake.h" role="src" />
<file baseinstalldir="/" name="src/core/security/json_token.h" role="src" />
@ -324,7 +325,7 @@
<file baseinstalldir="/" name="src/core/client_config/subchannel_factory.c" role="src" />
<file baseinstalldir="/" name="src/core/client_config/subchannel_index.c" role="src" />
<file baseinstalldir="/" name="src/core/client_config/uri_parser.c" role="src" />
<file baseinstalldir="/" name="src/core/compression/algorithm.c" role="src" />
<file baseinstalldir="/" name="src/core/compression/compression_algorithm.c" role="src" />
<file baseinstalldir="/" name="src/core/compression/message_compress.c" role="src" />
<file baseinstalldir="/" name="src/core/debug/trace.c" role="src" />
<file baseinstalldir="/" name="src/core/httpcli/format_request.c" role="src" />
@ -425,7 +426,7 @@
<file baseinstalldir="/" name="src/core/transport/transport.c" role="src" />
<file baseinstalldir="/" name="src/core/transport/transport_op_string.c" role="src" />
<file baseinstalldir="/" name="src/core/httpcli/httpcli_security_connector.c" role="src" />
<file baseinstalldir="/" name="src/core/security/base64.c" role="src" />
<file baseinstalldir="/" name="src/core/security/b64.c" role="src" />
<file baseinstalldir="/" name="src/core/security/client_auth_filter.c" role="src" />
<file baseinstalldir="/" name="src/core/security/credentials.c" role="src" />
<file baseinstalldir="/" name="src/core/security/credentials_metadata.c" role="src" />
@ -988,7 +989,7 @@ Update to wrap gRPC C Core version 0.10.0
<release>beta</release>
<api>beta</api>
</stability>
<date>2016-02-23</date>
<date>2016-02-24</date>
<license>BSD</license>
<notes>
- Simplify gRPC PHP installation #4517

31
tools/buildgen/generate_projects.py
Unescape View File

@ -45,12 +45,12 @@ import jobset
os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..'))
argp = argparse.ArgumentParser()
argp.add_argument('json', nargs='+')
argp.add_argument('build_files', nargs='+', default=[])
argp.add_argument('--templates', nargs='+', default=[])
argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int)
args = argp.parse_args()
json = args.json
json = args.build_files
test = {} if 'TEST' in os.environ else None
@ -62,21 +62,31 @@ if not templates:
for f in files:
templates.append(os.path.join(root, f))
pre_jobs = []
base_cmd = ['python2.7', 'tools/buildgen/mako_renderer.py']
cmd = base_cmd[:]
for plugin in plugins:
cmd.append('-p')
cmd.append(plugin)
for js in json:
cmd.append('-d')
cmd.append(js)
cmd.append('-w')
preprocessed_build = '.preprocessed_build'
cmd.append(preprocessed_build)
pre_jobs.append(jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None))
jobs = []
for template in templates:
for template in reversed(sorted(templates)):
root, f = os.path.split(template)
if os.path.splitext(f)[1] == '.template':
out_dir = '.' + root[len('templates'):]
out = out_dir + '/' + os.path.splitext(f)[0]
if not os.path.exists(out_dir):
os.makedirs(out_dir)
cmd = ['python2.7', 'tools/buildgen/mako_renderer.py']
for plugin in plugins:
cmd.append('-p')
cmd.append(plugin)
for js in json:
cmd.append('-d')
cmd.append(js)
cmd = base_cmd[:]
cmd.append('-P')
cmd.append(preprocessed_build)
cmd.append('-o')
if test is None:
cmd.append(out)
@ -88,6 +98,7 @@ for template in templates:
cmd.append(root + '/' + f)
jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None))
jobset.run(pre_jobs, maxjobs=args.jobs)
jobset.run(jobs, maxjobs=args.jobs)
if test is not None:

40
tools/buildgen/mako_renderer.py
Unescape View File

@ -38,6 +38,7 @@ Just a wrapper around the mako rendering library.
import getopt
import imp
import os
import cPickle as pickle
import shutil
import sys
@ -66,21 +67,23 @@ def out(msg):
def showhelp():
out('mako-renderer.py [-o out] [-m cache] [-d dict] [-d dict...] template')
out('mako-renderer.py [-o out] [-m cache] [-P preprocessed_input] [-d dict] [-d dict...]'
' [-t template] [-w preprocessed_output]')
def main(argv):
got_input = False
module_directory = None
preprocessed_output = None
dictionary = {}
json_dict = {}
got_output = False
output_file = sys.stdout
plugins = []
output_name = None
got_preprocessed_input = False
try:
opts, args = getopt.getopt(argv, 'hm:d:o:p:')
opts, args = getopt.getopt(argv, 'hm:d:o:p:t:P:w:')
except getopt.GetoptError:
out('Unknown option')
showhelp()
@ -104,18 +107,31 @@ def main(argv):
showhelp()
sys.exit(4)
module_directory = arg
elif opt == '-P':
assert not got_preprocessed_input
assert json_dict == {}
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'plugins')))
with open(arg, 'r') as dict_file:
dictionary = pickle.load(dict_file)
got_preprocessed_input = True
elif opt == '-d':
dict_file = open(arg, 'r')
bunch.merge_json(json_dict, yaml.load(dict_file.read()))
dict_file.close()
assert not got_preprocessed_input
with open(arg, 'r') as dict_file:
bunch.merge_json(json_dict, yaml.load(dict_file.read()))
elif opt == '-p':
plugins.append(import_plugin(arg))
elif opt == '-w':
preprocessed_output = arg
for plugin in plugins:
plugin.mako_plugin(json_dict)
if not got_preprocessed_input:
for plugin in plugins:
plugin.mako_plugin(json_dict)
for k, v in json_dict.items():
dictionary[k] = bunch.to_bunch(v)
for k, v in json_dict.items():
dictionary[k] = bunch.to_bunch(v)
if preprocessed_output:
with open(preprocessed_output, 'w') as dict_file:
pickle.dump(dictionary, dict_file)
cleared_dir = False
for arg in args:
@ -168,11 +184,9 @@ def main(argv):
with open(item_output_name, 'w') as output_file:
template.render_context(Context(output_file, **args))
if not got_input:
if not got_input and not preprocessed_output:
out('Got nothing to do')
showhelp()
output_file.close()
if __name__ == '__main__':
main(sys.argv[1:])

Write Preview
Loading…
Cancel
Save