From 5898847ddf64c8e14322aac57b10b8a3b23e4290 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 11 Dec 2017 16:43:54 -0800 Subject: [PATCH] yapf tools/codegen --- tools/codegen/core/gen_header_frame.py | 154 ++-- ..._registered_method_bad_client_test_body.py | 74 +- tools/codegen/core/gen_settings_ids.py | 189 ++--- tools/codegen/core/gen_static_metadata.py | 374 +++++----- tools/codegen/core/gen_stats_data.py | 705 +++++++++--------- tools/distrib/yapf_code.sh | 1 + 6 files changed, 777 insertions(+), 720 deletions(-) diff --git a/tools/codegen/core/gen_header_frame.py b/tools/codegen/core/gen_header_frame.py index 5375c14fd6b..7219d4dd866 100755 --- a/tools/codegen/core/gen_header_frame.py +++ b/tools/codegen/core/gen_header_frame.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Read from stdin a set of colon separated http headers: :path: /foo/bar content-type: application/grpc @@ -24,109 +23,118 @@ import json import sys import argparse + def append_never_indexed(payload_line, n, count, key, value): - payload_line.append(0x10) - assert(len(key) <= 126) - payload_line.append(len(key)) - payload_line.extend(ord(c) for c in key) - assert(len(value) <= 126) - payload_line.append(len(value)) - payload_line.extend(ord(c) for c in value) + payload_line.append(0x10) + assert (len(key) <= 126) + payload_line.append(len(key)) + payload_line.extend(ord(c) for c in key) + assert (len(value) <= 126) + payload_line.append(len(value)) + payload_line.extend(ord(c) for c in value) + def append_inc_indexed(payload_line, n, count, key, value): - payload_line.append(0x40) - assert(len(key) <= 126) - payload_line.append(len(key)) - payload_line.extend(ord(c) for c in key) - assert(len(value) <= 126) - payload_line.append(len(value)) - payload_line.extend(ord(c) for c in value) + payload_line.append(0x40) + assert (len(key) <= 126) + payload_line.append(len(key)) + payload_line.extend(ord(c) for c in key) + assert (len(value) <= 126) + payload_line.append(len(value)) + payload_line.extend(ord(c) for c in value) + def append_pre_indexed(payload_line, n, count, key, value): - payload_line.append(0x80 + 61 + count - n) + payload_line.append(0x80 + 61 + count - n) + _COMPRESSORS = { - 'never': append_never_indexed, - 'inc': append_inc_indexed, - 'pre': append_pre_indexed, + 'never': append_never_indexed, + 'inc': append_inc_indexed, + 'pre': append_pre_indexed, } argp = argparse.ArgumentParser('Generate header frames') -argp.add_argument('--set_end_stream', default=False, action='store_const', const=True) -argp.add_argument('--no_framing', default=False, action='store_const', const=True) -argp.add_argument('--compression', choices=sorted(_COMPRESSORS.keys()), default='never') +argp.add_argument( + '--set_end_stream', default=False, action='store_const', const=True) +argp.add_argument( + '--no_framing', default=False, action='store_const', const=True) +argp.add_argument( + '--compression', choices=sorted(_COMPRESSORS.keys()), default='never') argp.add_argument('--hex', default=False, action='store_const', const=True) args = argp.parse_args() # parse input, fill in vals vals = [] for line in sys.stdin: - line = line.strip() - if line == '': continue - if line[0] == '#': continue - key_tail, value = line[1:].split(':') - key = (line[0] + key_tail).strip() - value = value.strip() - vals.append((key, value)) + line = line.strip() + if line == '': continue + if line[0] == '#': continue + key_tail, value = line[1:].split(':') + key = (line[0] + key_tail).strip() + value = value.strip() + vals.append((key, value)) # generate frame payload binary data payload_bytes = [] if not args.no_framing: - payload_bytes.append([]) # reserve space for header + payload_bytes.append([]) # reserve space for header payload_len = 0 n = 0 for key, value in vals: - payload_line = [] - _COMPRESSORS[args.compression](payload_line, n, len(vals), key, value) - n += 1 - payload_len += len(payload_line) - payload_bytes.append(payload_line) + payload_line = [] + _COMPRESSORS[args.compression](payload_line, n, len(vals), key, value) + n += 1 + payload_len += len(payload_line) + payload_bytes.append(payload_line) # fill in header if not args.no_framing: - flags = 0x04 # END_HEADERS - if args.set_end_stream: - flags |= 0x01 # END_STREAM - payload_bytes[0].extend([ - (payload_len >> 16) & 0xff, - (payload_len >> 8) & 0xff, - (payload_len) & 0xff, - # header frame - 0x01, - # flags - flags, - # stream id - 0x00, - 0x00, - 0x00, - 0x01 - ]) + flags = 0x04 # END_HEADERS + if args.set_end_stream: + flags |= 0x01 # END_STREAM + payload_bytes[0].extend([ + (payload_len >> 16) & 0xff, + (payload_len >> 8) & 0xff, + (payload_len) & 0xff, + # header frame + 0x01, + # flags + flags, + # stream id + 0x00, + 0x00, + 0x00, + 0x01 + ]) hex_bytes = [ord(c) for c in "abcdefABCDEF0123456789"] + def esc_c(line): - out = "\"" - last_was_hex = False - for c in line: - if 32 <= c < 127: - if c in hex_bytes and last_was_hex: - out += "\"\"" - if c != ord('"'): - out += chr(c) - else: - out += "\\\"" - last_was_hex = False - else: - out += "\\x%02x" % c - last_was_hex = True - return out + "\"" + out = "\"" + last_was_hex = False + for c in line: + if 32 <= c < 127: + if c in hex_bytes and last_was_hex: + out += "\"\"" + if c != ord('"'): + out += chr(c) + else: + out += "\\\"" + last_was_hex = False + else: + out += "\\x%02x" % c + last_was_hex = True + return out + "\"" + # dump bytes if args.hex: - all_bytes = [] - for line in payload_bytes: - all_bytes.extend(line) - print '{%s}' % ', '.join('0x%02x' % c for c in all_bytes) + all_bytes = [] + for line in payload_bytes: + all_bytes.extend(line) + print '{%s}' % ', '.join('0x%02x' % c for c in all_bytes) else: - for line in payload_bytes: - print esc_c(line) + for line in payload_bytes: + print esc_c(line) diff --git a/tools/codegen/core/gen_server_registered_method_bad_client_test_body.py b/tools/codegen/core/gen_server_registered_method_bad_client_test_body.py index 8b5b618e8ea..fa87c974f97 100755 --- a/tools/codegen/core/gen_server_registered_method_bad_client_test_body.py +++ b/tools/codegen/core/gen_server_registered_method_bad_client_test_body.py @@ -14,48 +14,42 @@ # See the License for the specific language governing permissions and # limitations under the License. + def esc_c(line): - out = "\"" - last_was_hex = False - for c in line: - if 32 <= c < 127: - if c in hex_bytes and last_was_hex: - out += "\"\"" - if c != ord('"'): - out += chr(c) - else: - out += "\\\"" - last_was_hex = False - else: - out += "\\x%02x" % c - last_was_hex = True - return out + "\"" + out = "\"" + last_was_hex = False + for c in line: + if 32 <= c < 127: + if c in hex_bytes and last_was_hex: + out += "\"\"" + if c != ord('"'): + out += chr(c) + else: + out += "\\\"" + last_was_hex = False + else: + out += "\\x%02x" % c + last_was_hex = True + return out + "\"" + done = set() for message_length in range(0, 3): - for send_message_length in range(0, message_length + 1): - payload = [ - 0, - (message_length >> 24) & 0xff, - (message_length >> 16) & 0xff, - (message_length >> 8) & 0xff, - (message_length) & 0xff - ] + send_message_length * [0] - for frame_length in range(0, len(payload) + 1): - is_end = frame_length == len(payload) and send_message_length == message_length - frame = [ - (frame_length >> 16) & 0xff, - (frame_length >> 8) & 0xff, - (frame_length) & 0xff, - 0, - 1 if is_end else 0, - 0, 0, 0, 1 - ] + payload[0:frame_length] - text = esc_c(frame) - if text not in done: - print 'GRPC_RUN_BAD_CLIENT_TEST(verifier_%s, PFX_STR %s, %s);' % ( - 'succeeds' if is_end else 'fails', - text, - '0' if is_end else 'GRPC_BAD_CLIENT_DISCONNECT') - done.add(text) + for send_message_length in range(0, message_length + 1): + payload = [ + 0, (message_length >> 24) & 0xff, (message_length >> 16) & 0xff, + (message_length >> 8) & 0xff, (message_length) & 0xff + ] + send_message_length * [0] + for frame_length in range(0, len(payload) + 1): + is_end = frame_length == len( + payload) and send_message_length == message_length + frame = [(frame_length >> 16) & 0xff, (frame_length >> 8) & 0xff, + (frame_length) & 0xff, 0, 1 + if is_end else 0, 0, 0, 0, 1] + payload[0:frame_length] + text = esc_c(frame) + if text not in done: + print 'GRPC_RUN_BAD_CLIENT_TEST(verifier_%s, PFX_STR %s, %s);' % ( + 'succeeds' if is_end else 'fails', text, '0' + if is_end else 'GRPC_BAD_CLIENT_DISCONNECT') + done.add(text) diff --git a/tools/codegen/core/gen_settings_ids.py b/tools/codegen/core/gen_settings_ids.py index 481c4216a31..bc4380605a6 100755 --- a/tools/codegen/core/gen_settings_ids.py +++ b/tools/codegen/core/gen_settings_ids.py @@ -24,92 +24,114 @@ Setting = collections.namedtuple('Setting', 'id default min max on_error') OnError = collections.namedtuple('OnError', 'behavior code') clamp_invalid_value = OnError('CLAMP_INVALID_VALUE', 'PROTOCOL_ERROR') disconnect_on_invalid_value = lambda e: OnError('DISCONNECT_ON_INVALID_VALUE', e) -DecoratedSetting = collections.namedtuple('DecoratedSetting', 'enum name setting') +DecoratedSetting = collections.namedtuple('DecoratedSetting', + 'enum name setting') _SETTINGS = { - 'HEADER_TABLE_SIZE': Setting(1, 4096, 0, 0xffffffff, clamp_invalid_value), - 'ENABLE_PUSH': Setting(2, 1, 0, 1, disconnect_on_invalid_value('PROTOCOL_ERROR')), - 'MAX_CONCURRENT_STREAMS': Setting(3, 0xffffffff, 0, 0xffffffff, disconnect_on_invalid_value('PROTOCOL_ERROR')), - 'INITIAL_WINDOW_SIZE': Setting(4, 65535, 0, 0x7fffffff, disconnect_on_invalid_value('FLOW_CONTROL_ERROR')), - 'MAX_FRAME_SIZE': Setting(5, 16384, 16384, 16777215, disconnect_on_invalid_value('PROTOCOL_ERROR')), - 'MAX_HEADER_LIST_SIZE': Setting(6, _MAX_HEADER_LIST_SIZE, 0, _MAX_HEADER_LIST_SIZE, clamp_invalid_value), - 'GRPC_ALLOW_TRUE_BINARY_METADATA': Setting(0xfe03, 0, 0, 1, clamp_invalid_value), + 'HEADER_TABLE_SIZE': + Setting(1, 4096, 0, 0xffffffff, clamp_invalid_value), + 'ENABLE_PUSH': + Setting(2, 1, 0, 1, disconnect_on_invalid_value('PROTOCOL_ERROR')), + 'MAX_CONCURRENT_STREAMS': + Setting(3, 0xffffffff, 0, 0xffffffff, + disconnect_on_invalid_value('PROTOCOL_ERROR')), + 'INITIAL_WINDOW_SIZE': + Setting(4, 65535, 0, 0x7fffffff, + disconnect_on_invalid_value('FLOW_CONTROL_ERROR')), + 'MAX_FRAME_SIZE': + Setting(5, 16384, 16384, 16777215, + disconnect_on_invalid_value('PROTOCOL_ERROR')), + 'MAX_HEADER_LIST_SIZE': + Setting(6, _MAX_HEADER_LIST_SIZE, 0, _MAX_HEADER_LIST_SIZE, + clamp_invalid_value), + 'GRPC_ALLOW_TRUE_BINARY_METADATA': + Setting(0xfe03, 0, 0, 1, clamp_invalid_value), } H = open('src/core/ext/transport/chttp2/transport/http2_settings.h', 'w') C = open('src/core/ext/transport/chttp2/transport/http2_settings.c', 'w') + # utility: print a big comment block into a set of files def put_banner(files, banner): - for f in files: - print >>f, '/*' - for line in banner: - print >>f, ' * %s' % line - print >>f, ' */' - print >>f + for f in files: + print >> f, '/*' + for line in banner: + print >> f, ' * %s' % line + print >> f, ' */' + print >> f + # copy-paste copyright notice from this file with open(sys.argv[0]) as my_source: - copyright = [] - for line in my_source: - if line[0] != '#': break - for line in my_source: - if line[0] == '#': - copyright.append(line) - break - for line in my_source: - if line[0] != '#': - break - copyright.append(line) - put_banner([H,C], [line[2:].rstrip() for line in copyright]) - -put_banner([H,C], ["Automatically generated by tools/codegen/core/gen_settings_ids.py"]) - -print >>H, "#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H" -print >>H, "#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H" -print >>H -print >>H, "#include " -print >>H, "#include " -print >>H - -print >>C, "#include \"src/core/ext/transport/chttp2/transport/http2_settings.h\"" -print >>C -print >>C, "#include " -print >>C, "#include \"src/core/lib/transport/http2_errors.h\"" -print >>C + copyright = [] + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + copyright.append(line) + break + for line in my_source: + if line[0] != '#': + break + copyright.append(line) + put_banner([H, C], [line[2:].rstrip() for line in copyright]) + +put_banner( + [H, C], + ["Automatically generated by tools/codegen/core/gen_settings_ids.py"]) + +print >> H, "#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H" +print >> H, "#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H" +print >> H +print >> H, "#include " +print >> H, "#include " +print >> H + +print >> C, "#include \"src/core/ext/transport/chttp2/transport/http2_settings.h\"" +print >> C +print >> C, "#include " +print >> C, "#include \"src/core/lib/transport/http2_errors.h\"" +print >> C p = perfection.hash_parameters(sorted(x.id for x in _SETTINGS.values())) print p + def hash(i): - i += p.offset - x = i % p.t - y = i / p.t - return x + p.r[y] + i += p.offset + x = i % p.t + y = i / p.t + return x + p.r[y] + -decorated_settings = [DecoratedSetting(hash(setting.id), name, setting) - for name, setting in _SETTINGS.iteritems()] +decorated_settings = [ + DecoratedSetting(hash(setting.id), name, setting) + for name, setting in _SETTINGS.iteritems() +] -print >>H, 'typedef enum {' +print >> H, 'typedef enum {' for decorated_setting in sorted(decorated_settings): - print >>H, ' GRPC_CHTTP2_SETTINGS_%s = %d, /* wire id %d */' % ( - decorated_setting.name, decorated_setting.enum, decorated_setting.setting.id) -print >>H, '} grpc_chttp2_setting_id;' -print >>H -print >>H, '#define GRPC_CHTTP2_NUM_SETTINGS %d' % (max(x.enum for x in decorated_settings) + 1) - -print >>H, 'extern const uint16_t grpc_setting_id_to_wire_id[];' -print >>C, 'const uint16_t grpc_setting_id_to_wire_id[] = {%s};' % ','.join( + print >> H, ' GRPC_CHTTP2_SETTINGS_%s = %d, /* wire id %d */' % ( + decorated_setting.name, decorated_setting.enum, + decorated_setting.setting.id) +print >> H, '} grpc_chttp2_setting_id;' +print >> H +print >> H, '#define GRPC_CHTTP2_NUM_SETTINGS %d' % ( + max(x.enum for x in decorated_settings) + 1) + +print >> H, 'extern const uint16_t grpc_setting_id_to_wire_id[];' +print >> C, 'const uint16_t grpc_setting_id_to_wire_id[] = {%s};' % ','.join( '%d' % s for s in p.slots) -print >>H -print >>H, "bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out);" +print >> H +print >> H, "bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out);" cgargs = { - 'r': ','.join('%d' % (r if r is not None else 0) for r in p.r), - 't': p.t, - 'offset': abs(p.offset), - 'offset_sign': '+' if p.offset > 0 else '-' - } -print >>C, """ + 'r': ','.join('%d' % (r if r is not None else 0) for r in p.r), + 't': p.t, + 'offset': abs(p.offset), + 'offset_sign': '+' if p.offset > 0 else '-' +} +print >> C, """ bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out) { uint32_t i = wire_id %(offset_sign)s %(offset)d; uint32_t x = i %% %(t)d; @@ -118,17 +140,17 @@ bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out) { switch (y) { """ % cgargs for i, r in enumerate(p.r): - if not r: continue - if r < 0: print >>C, 'case %d: h -= %d; break;' % (i, -r) - else: print >>C, 'case %d: h += %d; break;' % (i, r) -print >>C, """ + if not r: continue + if r < 0: print >> C, 'case %d: h -= %d; break;' % (i, -r) + else: print >> C, 'case %d: h += %d; break;' % (i, r) +print >> C, """ } *out = (grpc_chttp2_setting_id)h; return h < GPR_ARRAY_SIZE(grpc_setting_id_to_wire_id) && grpc_setting_id_to_wire_id[h] == wire_id; } """ % cgargs -print >>H, """ +print >> H, """ typedef enum { GRPC_CHTTP2_CLAMP_INVALID_VALUE, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE @@ -145,25 +167,22 @@ typedef struct { extern const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS]; """ -print >>C, "const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS] = {" +print >> C, "const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS] = {" i = 0 for decorated_setting in sorted(decorated_settings): - while i < decorated_setting.enum: - print >>C, "{NULL, 0, 0, 0, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_HTTP2_PROTOCOL_ERROR}," + while i < decorated_setting.enum: + print >> C, "{NULL, 0, 0, 0, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_HTTP2_PROTOCOL_ERROR}," + i += 1 + print >> C, "{\"%s\", %du, %du, %du, GRPC_CHTTP2_%s, GRPC_HTTP2_%s}," % ( + decorated_setting.name, decorated_setting.setting.default, + decorated_setting.setting.min, decorated_setting.setting.max, + decorated_setting.setting.on_error.behavior, + decorated_setting.setting.on_error.code,) i += 1 - print >>C, "{\"%s\", %du, %du, %du, GRPC_CHTTP2_%s, GRPC_HTTP2_%s}," % ( - decorated_setting.name, - decorated_setting.setting.default, - decorated_setting.setting.min, - decorated_setting.setting.max, - decorated_setting.setting.on_error.behavior, - decorated_setting.setting.on_error.code, - ) - i += 1 -print >>C, "};" - -print >>H -print >>H, "#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H */" +print >> C, "};" + +print >> H +print >> H, "#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H */" H.close() C.close() diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index 7847b8ed1de..c73cdc08161 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -172,65 +172,66 @@ STREAM_COMPRESSION_ALGORITHMS = [ 'gzip', ] + # utility: mangle the name of a config def mangle(elem, name=None): - xl = { - '-': '_', - ':': '', - '/': 'slash', - '.': 'dot', - ',': 'comma', - ' ': '_', - } + xl = { + '-': '_', + ':': '', + '/': 'slash', + '.': 'dot', + ',': 'comma', + ' ': '_', + } - def m0(x): - if not x: - return 'empty' - r = '' - for c in x: - put = xl.get(c, c.lower()) - if not put: - continue - last_is_underscore = r[-1] == '_' if r else True - if last_is_underscore and put == '_': - continue - elif len(put) > 1: - if not last_is_underscore: - r += '_' - r += put - r += '_' - else: - r += put - if r[-1] == '_': - r = r[:-1] - return r - - def n(default, name=name): - if name is None: - return 'grpc_%s_' % default - if name == '': - return '' - return 'grpc_%s_' % name - - if isinstance(elem, tuple): - return '%s%s_%s' % (n('mdelem'), m0(elem[0]), m0(elem[1])) - else: - return '%s%s' % (n('mdstr'), m0(elem)) + def m0(x): + if not x: + return 'empty' + r = '' + for c in x: + put = xl.get(c, c.lower()) + if not put: + continue + last_is_underscore = r[-1] == '_' if r else True + if last_is_underscore and put == '_': + continue + elif len(put) > 1: + if not last_is_underscore: + r += '_' + r += put + r += '_' + else: + r += put + if r[-1] == '_': + r = r[:-1] + return r + + def n(default, name=name): + if name is None: + return 'grpc_%s_' % default + if name == '': + return '' + return 'grpc_%s_' % name + + if isinstance(elem, tuple): + return '%s%s_%s' % (n('mdelem'), m0(elem[0]), m0(elem[1])) + else: + return '%s%s' % (n('mdstr'), m0(elem)) # utility: generate some hash value for a string def fake_hash(elem): - return hashlib.md5(elem).hexdigest()[0:8] + return hashlib.md5(elem).hexdigest()[0:8] # utility: print a big comment block into a set of files def put_banner(files, banner): - for f in files: - print >> f, '/*' - for line in banner: - print >> f, ' * %s' % line - print >> f, ' */' - print >> f + for f in files: + print >> f, '/*' + for line in banner: + print >> f, ' * %s' % line + print >> f, ' */' + print >> f # build a list of all the strings we need @@ -240,43 +241,43 @@ static_userdata = {} # put metadata batch callouts first, to make the check of if a static metadata # string is a callout trivial for elem, _ in METADATA_BATCH_CALLOUTS: - if elem not in all_strs: - all_strs.append(elem) -for elem in CONFIG: - if isinstance(elem, tuple): - if elem[0] not in all_strs: - all_strs.append(elem[0]) - if elem[1] not in all_strs: - all_strs.append(elem[1]) - if elem not in all_elems: - all_elems.append(elem) - else: if elem not in all_strs: - all_strs.append(elem) + all_strs.append(elem) +for elem in CONFIG: + if isinstance(elem, tuple): + if elem[0] not in all_strs: + all_strs.append(elem[0]) + if elem[1] not in all_strs: + all_strs.append(elem[1]) + if elem not in all_elems: + all_elems.append(elem) + else: + if elem not in all_strs: + all_strs.append(elem) compression_elems = [] for mask in range(1, 1 << len(COMPRESSION_ALGORITHMS)): - val = ','.join(COMPRESSION_ALGORITHMS[alg] - for alg in range(0, len(COMPRESSION_ALGORITHMS)) - if (1 << alg) & mask) - elem = ('grpc-accept-encoding', val) - if val not in all_strs: - all_strs.append(val) - if elem not in all_elems: - all_elems.append(elem) - compression_elems.append(elem) - static_userdata[elem] = 1 + (mask | 1) + val = ','.join(COMPRESSION_ALGORITHMS[alg] + for alg in range(0, len(COMPRESSION_ALGORITHMS)) + if (1 << alg) & mask) + elem = ('grpc-accept-encoding', val) + if val not in all_strs: + all_strs.append(val) + if elem not in all_elems: + all_elems.append(elem) + compression_elems.append(elem) + static_userdata[elem] = 1 + (mask | 1) stream_compression_elems = [] for mask in range(1, 1 << len(STREAM_COMPRESSION_ALGORITHMS)): - val = ','.join(STREAM_COMPRESSION_ALGORITHMS[alg] - for alg in range(0, len(STREAM_COMPRESSION_ALGORITHMS)) - if (1 << alg) & mask) - elem = ('accept-encoding', val) - if val not in all_strs: - all_strs.append(val) - if elem not in all_elems: - all_elems.append(elem) - stream_compression_elems.append(elem) - static_userdata[elem] = 1 + (mask | 1) + val = ','.join(STREAM_COMPRESSION_ALGORITHMS[alg] + for alg in range(0, len(STREAM_COMPRESSION_ALGORITHMS)) + if (1 << alg) & mask) + elem = ('accept-encoding', val) + if val not in all_strs: + all_strs.append(val) + if elem not in all_elems: + all_elems.append(elem) + stream_compression_elems.append(elem) + static_userdata[elem] = 1 + (mask | 1) # output configuration args = sys.argv[1:] @@ -284,62 +285,62 @@ H = None C = None D = None if args: - if 'header' in args: - H = sys.stdout - else: - H = open('/dev/null', 'w') - if 'source' in args: - C = sys.stdout - else: - C = open('/dev/null', 'w') - if 'dictionary' in args: - D = sys.stdout - else: - D = open('/dev/null', 'w') + if 'header' in args: + H = sys.stdout + else: + H = open('/dev/null', 'w') + if 'source' in args: + C = sys.stdout + else: + C = open('/dev/null', 'w') + if 'dictionary' in args: + D = sys.stdout + else: + D = open('/dev/null', 'w') else: - H = open( - os.path.join( - os.path.dirname(sys.argv[0]), - '../../../src/core/lib/transport/static_metadata.h'), 'w') - C = open( - os.path.join( - os.path.dirname(sys.argv[0]), - '../../../src/core/lib/transport/static_metadata.cc'), 'w') - D = open( - os.path.join( - os.path.dirname(sys.argv[0]), - '../../../test/core/end2end/fuzzers/hpack.dictionary'), 'w') + H = open( + os.path.join( + os.path.dirname(sys.argv[0]), + '../../../src/core/lib/transport/static_metadata.h'), 'w') + C = open( + os.path.join( + os.path.dirname(sys.argv[0]), + '../../../src/core/lib/transport/static_metadata.cc'), 'w') + D = open( + os.path.join( + os.path.dirname(sys.argv[0]), + '../../../test/core/end2end/fuzzers/hpack.dictionary'), 'w') # copy-paste copyright notice from this file with open(sys.argv[0]) as my_source: - copyright = [] - for line in my_source: - if line[0] != '#': - break - for line in my_source: - if line[0] == '#': - copyright.append(line) - break - for line in my_source: - if line[0] != '#': - break - copyright.append(line) - put_banner([H, C], [line[2:].rstrip() for line in copyright]) + copyright = [] + for line in my_source: + if line[0] != '#': + break + for line in my_source: + if line[0] == '#': + copyright.append(line) + break + for line in my_source: + if line[0] != '#': + break + copyright.append(line) + put_banner([H, C], [line[2:].rstrip() for line in copyright]) hex_bytes = [ord(c) for c in 'abcdefABCDEF0123456789'] def esc_dict(line): - out = "\"" - for c in line: - if 32 <= c < 127: - if c != ord('"'): - out += chr(c) - else: - out += "\\\"" - else: - out += '\\x%02X' % c - return out + "\"" + out = "\"" + for c in line: + if 32 <= c < 127: + if c != ord('"'): + out += chr(c) + else: + out += "\\\"" + else: + out += '\\x%02X' % c + return out + "\"" put_banner([H, C], """WARNING: Auto-generated code. @@ -369,27 +370,26 @@ print >> C str_ofs = 0 id2strofs = {} for i, elem in enumerate(all_strs): - id2strofs[i] = str_ofs - str_ofs += len(elem) + id2strofs[i] = str_ofs + str_ofs += len(elem) def slice_def(i): - return ('{&grpc_static_metadata_refcounts[%d],' - ' {{g_bytes+%d, %d}}}') % ( - i, id2strofs[i], len(all_strs[i])) + return ('{&grpc_static_metadata_refcounts[%d],' + ' {{g_bytes+%d, %d}}}') % (i, id2strofs[i], len(all_strs[i])) # validate configuration for elem, _ in METADATA_BATCH_CALLOUTS: - assert elem in all_strs + assert elem in all_strs print >> H, '#define GRPC_STATIC_MDSTR_COUNT %d' % len(all_strs) print >> H, ('extern const grpc_slice ' 'grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT];') for i, elem in enumerate(all_strs): - print >> H, '/* "%s" */' % elem - print >> H, '#define %s (grpc_static_slice_table[%d])' % ( - mangle(elem).upper(), i) + print >> H, '/* "%s" */' % elem + print >> H, '#define %s (grpc_static_slice_table[%d])' % ( + mangle(elem).upper(), i) print >> H print >> C, 'static uint8_t g_bytes[] = {%s};' % ( ','.join('%d' % ord(c) for c in ''.join(all_strs))) @@ -411,7 +411,7 @@ print >> H, ('extern grpc_slice_refcount ' print >> C, ('grpc_slice_refcount ' 'grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = {') for i, elem in enumerate(all_strs): - print >> C, ' {&grpc_static_metadata_vtable, &static_sub_refcnt},' + print >> C, ' {&grpc_static_metadata_vtable, &static_sub_refcnt},' print >> C, '};' print >> C print >> H, '#define GRPC_IS_STATIC_METADATA_STRING(slice) \\' @@ -421,7 +421,7 @@ print >> H print >> C, ('const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]' ' = {') for i, elem in enumerate(all_strs): - print >> C, slice_def(i) + ',' + print >> C, slice_def(i) + ',' print >> C, '};' print >> C print >> H, '#define GRPC_STATIC_METADATA_INDEX(static_slice) \\' @@ -431,10 +431,10 @@ print >> H print >> D, '# hpack fuzzing dictionary' for i, elem in enumerate(all_strs): - print >> D, '%s' % (esc_dict([len(elem)] + [ord(c) for c in elem])) + print >> D, '%s' % (esc_dict([len(elem)] + [ord(c) for c in elem])) for i, elem in enumerate(all_elems): - print >> D, '%s' % (esc_dict([0, len(elem[0])] + [ord(c) for c in elem[0]] + - [len(elem[1])] + [ord(c) for c in elem[1]])) + print >> D, '%s' % (esc_dict([0, len(elem[0])] + [ord(c) for c in elem[0]] + + [len(elem[1])] + [ord(c) for c in elem[1]])) print >> H, '#define GRPC_STATIC_MDELEM_COUNT %d' % len(all_elems) print >> H, ('extern grpc_mdelem_data ' @@ -442,10 +442,9 @@ print >> H, ('extern grpc_mdelem_data ' print >> H, ('extern uintptr_t ' 'grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];') for i, elem in enumerate(all_elems): - print >> H, '/* "%s": "%s" */' % elem - print >> H, ('#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d], ' - 'GRPC_MDELEM_STORAGE_STATIC))') % ( - mangle(elem).upper(), i) + print >> H, '/* "%s": "%s" */' % elem + print >> H, ('#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d], ' + 'GRPC_MDELEM_STORAGE_STATIC))') % (mangle(elem).upper(), i) print >> H print >> C, ('uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] ' '= {') @@ -456,42 +455,38 @@ print >> C def str_idx(s): - for i, s2 in enumerate(all_strs): - if s == s2: - return i + for i, s2 in enumerate(all_strs): + if s == s2: + return i def md_idx(m): - for i, m2 in enumerate(all_elems): - if m == m2: - return i + for i, m2 in enumerate(all_elems): + if m == m2: + return i def offset_trials(mink): - yield 0 - for i in range(1, 100): - for mul in [-1, 1]: - yield mul * i + yield 0 + for i in range(1, 100): + for mul in [-1, 1]: + yield mul * i def perfect_hash(keys, name): - p = perfection.hash_parameters(keys) - - def f(i, p=p): - i += p.offset - x = i % p.t - y = i / p.t - return x + p.r[y] - - return { - 'PHASHRANGE': - p.t - 1 + max(p.r), - 'PHASHNKEYS': - len(p.slots), - 'pyfunc': - f, - 'code': - """ + p = perfection.hash_parameters(keys) + + def f(i, p=p): + i += p.offset + x = i % p.t + y = i / p.t + return x + p.r[y] + + return { + 'PHASHRANGE': p.t - 1 + max(p.r), + 'PHASHNKEYS': len(p.slots), + 'pyfunc': f, + 'code': """ static const int8_t %(name)s_r[] = {%(r)s}; static uint32_t %(name)s_phash(uint32_t i) { i %(offset_sign)s= %(offset)d; @@ -505,13 +500,13 @@ static uint32_t %(name)s_phash(uint32_t i) { return h; } """ % { - 'name': name, - 'r': ','.join('%d' % (r if r is not None else 0) for r in p.r), - 't': p.t, - 'offset': abs(p.offset), - 'offset_sign': '+' if p.offset > 0 else '-' + 'name': name, + 'r': ','.join('%d' % (r if r is not None else 0) for r in p.r), + 't': p.t, + 'offset': abs(p.offset), + 'offset_sign': '+' if p.offset > 0 else '-' + } } - } elem_keys = [ @@ -523,14 +518,14 @@ print >> C, elem_hash['code'] keys = [0] * int(elem_hash['PHASHRANGE']) idxs = [255] * int(elem_hash['PHASHNKEYS']) for i, k in enumerate(elem_keys): - h = elem_hash['pyfunc'](k) - assert keys[h] == 0 - keys[h] = k - idxs[h] = i + h = elem_hash['pyfunc'](k) + assert keys[h] == 0 + keys[h] = k + idxs[h] = i print >> C, 'static const uint16_t elem_keys[] = {%s};' % ','.join( '%d' % k for k in keys) -print >> C, 'static const uint8_t elem_idxs[] = {%s};' % ','.join( - '%d' % i for i in idxs) +print >> C, 'static const uint8_t elem_idxs[] = {%s};' % ','.join('%d' % i + for i in idxs) print >> C print >> H, 'grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b);' @@ -544,12 +539,12 @@ print >> C print >> C, 'grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = {' for a, b in all_elems: - print >> C, '{%s,%s},' % (slice_def(str_idx(a)), slice_def(str_idx(b))) + print >> C, '{%s,%s},' % (slice_def(str_idx(a)), slice_def(str_idx(b))) print >> C, '};' print >> H, 'typedef enum {' for elem, _ in METADATA_BATCH_CALLOUTS: - print >> H, ' %s,' % mangle(elem, 'batch').upper() + print >> H, ' %s,' % mangle(elem, 'batch').upper() print >> H, ' GRPC_BATCH_CALLOUTS_COUNT' print >> H, '} grpc_metadata_batch_callouts_index;' print >> H @@ -557,7 +552,7 @@ print >> H, 'typedef union {' print >> H, ' struct grpc_linked_mdelem *array[GRPC_BATCH_CALLOUTS_COUNT];' print >> H, ' struct {' for elem, _ in METADATA_BATCH_CALLOUTS: - print >> H, ' struct grpc_linked_mdelem *%s;' % mangle(elem, '').lower() + print >> H, ' struct grpc_linked_mdelem *%s;' % mangle(elem, '').lower() print >> H, ' } named;' print >> H, '} grpc_metadata_batch_callouts;' print >> H @@ -569,7 +564,7 @@ print >> H, ('extern bool grpc_static_callout_is_default[' print >> H print >> C, 'bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = {' for elem, is_default in METADATA_BATCH_CALLOUTS: - print >> C, ' %s, // %s' % (str(is_default).lower(), elem) + print >> C, ' %s, // %s' % (str(is_default).lower(), elem) print >> C, '};' print >> C @@ -588,7 +583,8 @@ print >> H, 'extern const uint8_t grpc_static_accept_stream_encoding_metadata[%d 1 << len(STREAM_COMPRESSION_ALGORITHMS)) print >> C, 'const uint8_t grpc_static_accept_stream_encoding_metadata[%d] = {' % ( 1 << len(STREAM_COMPRESSION_ALGORITHMS)) -print >> C, '0,%s' % ','.join('%d' % md_idx(elem) for elem in stream_compression_elems) +print >> C, '0,%s' % ','.join('%d' % md_idx(elem) + for elem in stream_compression_elems) print >> C, '};' print >> H, '#define GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS(algs) (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[grpc_static_accept_stream_encoding_metadata[(algs)]], GRPC_MDELEM_STORAGE_STATIC))' diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index d439d99a8c6..4fd9fa437e3 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -22,397 +22,436 @@ import yaml import json with open('src/core/lib/debug/stats_data.yaml') as f: - attrs = yaml.load(f.read()) + attrs = yaml.load(f.read()) REQUIRED_FIELDS = ['name', 'doc'] + def make_type(name, fields): - return (collections.namedtuple(name, ' '.join(list(set(REQUIRED_FIELDS + fields)))), []) + return (collections.namedtuple( + name, ' '.join(list(set(REQUIRED_FIELDS + fields)))), []) + def c_str(s, encoding='ascii'): - if isinstance(s, unicode): - s = s.encode(encoding) - result = '' - for c in s: - if not (32 <= ord(c) < 127) or c in ('\\', '"'): - result += '\\%03o' % ord(c) - else: - result += c - return '"' + result + '"' - -types = ( - make_type('Counter', []), - make_type('Histogram', ['max', 'buckets']), -) + if isinstance(s, unicode): + s = s.encode(encoding) + result = '' + for c in s: + if not (32 <= ord(c) < 127) or c in ('\\', '"'): + result += '\\%03o' % ord(c) + else: + result += c + return '"' + result + '"' + + +types = (make_type('Counter', []), make_type('Histogram', ['max', 'buckets']),) inst_map = dict((t[0].__name__, t[1]) for t in types) stats = [] for attr in attrs: - found = False - for t, lst in types: - t_name = t.__name__.lower() - if t_name in attr: - name = attr[t_name] - del attr[t_name] - lst.append(t(name=name, **attr)) - found = True - break - assert found, "Bad decl: %s" % attr + found = False + for t, lst in types: + t_name = t.__name__.lower() + if t_name in attr: + name = attr[t_name] + del attr[t_name] + lst.append(t(name=name, **attr)) + found = True + break + assert found, "Bad decl: %s" % attr + def dbl2u64(d): - return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value + return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value + def shift_works_until(mapped_bounds, shift_bits): - for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])): - a, b = ab - if (a >> shift_bits) == (b >> shift_bits): - return i - return len(mapped_bounds) + for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])): + a, b = ab + if (a >> shift_bits) == (b >> shift_bits): + return i + return len(mapped_bounds) + def find_ideal_shift(mapped_bounds, max_size): - best = None - for shift_bits in reversed(range(0,64)): - n = shift_works_until(mapped_bounds, shift_bits) - if n == 0: continue - table_size = mapped_bounds[n-1] >> shift_bits - if table_size > max_size: continue - if table_size > 65535: continue - if best is None: - best = (shift_bits, n, table_size) - elif best[1] < n: - best = (shift_bits, n, table_size) - print best - return best + best = None + for shift_bits in reversed(range(0, 64)): + n = shift_works_until(mapped_bounds, shift_bits) + if n == 0: continue + table_size = mapped_bounds[n - 1] >> shift_bits + if table_size > max_size: continue + if table_size > 65535: continue + if best is None: + best = (shift_bits, n, table_size) + elif best[1] < n: + best = (shift_bits, n, table_size) + print best + return best + def gen_map_table(mapped_bounds, shift_data): - tbl = [] - cur = 0 - print mapped_bounds - mapped_bounds = [x >> shift_data[0] for x in mapped_bounds] - print mapped_bounds - for i in range(0, mapped_bounds[shift_data[1]-1]): - while i > mapped_bounds[cur]: - cur += 1 - tbl.append(cur) - return tbl + tbl = [] + cur = 0 + print mapped_bounds + mapped_bounds = [x >> shift_data[0] for x in mapped_bounds] + print mapped_bounds + for i in range(0, mapped_bounds[shift_data[1] - 1]): + while i > mapped_bounds[cur]: + cur += 1 + tbl.append(cur) + return tbl + static_tables = [] + def decl_static_table(values, type): - global static_tables - v = (type, values) - for i, vp in enumerate(static_tables): - if v == vp: return i - print "ADD TABLE: %s %r" % (type, values) - r = len(static_tables) - static_tables.append(v) - return r + global static_tables + v = (type, values) + for i, vp in enumerate(static_tables): + if v == vp: return i + print "ADD TABLE: %s %r" % (type, values) + r = len(static_tables) + static_tables.append(v) + return r + def type_for_uint_table(table): - mv = max(table) - if mv < 2**8: - return 'uint8_t' - elif mv < 2**16: - return 'uint16_t' - elif mv < 2**32: - return 'uint32_t' - else: - return 'uint64_t' + mv = max(table) + if mv < 2**8: + return 'uint8_t' + elif mv < 2**16: + return 'uint16_t' + elif mv < 2**32: + return 'uint32_t' + else: + return 'uint64_t' + def gen_bucket_code(histogram): - bounds = [0, 1] - done_trivial = False - done_unmapped = False - first_nontrivial = None - first_unmapped = None - while len(bounds) < histogram.buckets + 1: - if len(bounds) == histogram.buckets: - nextb = int(histogram.max) + bounds = [0, 1] + done_trivial = False + done_unmapped = False + first_nontrivial = None + first_unmapped = None + while len(bounds) < histogram.buckets + 1: + if len(bounds) == histogram.buckets: + nextb = int(histogram.max) + else: + mul = math.pow( + float(histogram.max) / bounds[-1], + 1.0 / (histogram.buckets + 1 - len(bounds))) + nextb = int(math.ceil(bounds[-1] * mul)) + if nextb <= bounds[-1] + 1: + nextb = bounds[-1] + 1 + elif not done_trivial: + done_trivial = True + first_nontrivial = len(bounds) + bounds.append(nextb) + bounds_idx = decl_static_table(bounds, 'int') + if done_trivial: + first_nontrivial_code = dbl2u64(first_nontrivial) + code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds] + shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * + histogram.buckets) + #print first_nontrivial, shift_data, bounds + #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]] + code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max + map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data) + if first_nontrivial is None: + code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' % + histogram.name.upper()) else: - mul = math.pow(float(histogram.max) / bounds[-1], - 1.0 / (histogram.buckets + 1 - len(bounds))) - nextb = int(math.ceil(bounds[-1] * mul)) - if nextb <= bounds[-1] + 1: - nextb = bounds[-1] + 1 - elif not done_trivial: - done_trivial = True - first_nontrivial = len(bounds) - bounds.append(nextb) - bounds_idx = decl_static_table(bounds, 'int') - if done_trivial: - first_nontrivial_code = dbl2u64(first_nontrivial) - code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds] - shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets) - #print first_nontrivial, shift_data, bounds - #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]] - code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max - map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data) - if first_nontrivial is None: - code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' - % histogram.name.upper()) - else: - code += 'if (value < %d) {\n' % first_nontrivial - code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' - % histogram.name.upper()) - code += 'return;\n' - code += '}' - first_nontrivial_code = dbl2u64(first_nontrivial) - if shift_data is not None: - map_table_idx = decl_static_table(map_table, type_for_uint_table(map_table)) - code += 'union { double dbl; uint64_t uint; } _val, _bkt;\n' - code += '_val.dbl = value;\n' - code += 'if (_val.uint < %dull) {\n' % ((map_table[-1] << shift_data[0]) + first_nontrivial_code) - code += 'int bucket = ' - code += 'grpc_stats_table_%d[((_val.uint - %dull) >> %d)] + %d;\n' % (map_table_idx, first_nontrivial_code, shift_data[0], first_nontrivial) - code += '_bkt.dbl = grpc_stats_table_%d[bucket];\n' % bounds_idx - code += 'bucket -= (_val.uint < _bkt.uint);\n' - code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, bucket);\n' % histogram.name.upper() - code += 'return;\n' - code += '}\n' - code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, '% histogram.name.upper() - code += 'grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_%d, %d));\n' % (bounds_idx, histogram.buckets) - return (code, bounds_idx) + code += 'if (value < %d) {\n' % first_nontrivial + code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' % + histogram.name.upper()) + code += 'return;\n' + code += '}' + first_nontrivial_code = dbl2u64(first_nontrivial) + if shift_data is not None: + map_table_idx = decl_static_table(map_table, + type_for_uint_table(map_table)) + code += 'union { double dbl; uint64_t uint; } _val, _bkt;\n' + code += '_val.dbl = value;\n' + code += 'if (_val.uint < %dull) {\n' % ( + (map_table[-1] << shift_data[0]) + first_nontrivial_code) + code += 'int bucket = ' + code += 'grpc_stats_table_%d[((_val.uint - %dull) >> %d)] + %d;\n' % ( + map_table_idx, first_nontrivial_code, shift_data[0], + first_nontrivial) + code += '_bkt.dbl = grpc_stats_table_%d[bucket];\n' % bounds_idx + code += 'bucket -= (_val.uint < _bkt.uint);\n' + code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, bucket);\n' % histogram.name.upper( + ) + code += 'return;\n' + code += '}\n' + code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, ' % histogram.name.upper( + ) + code += 'grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_%d, %d));\n' % ( + bounds_idx, histogram.buckets) + return (code, bounds_idx) + # utility: print a big comment block into a set of files def put_banner(files, banner): - for f in files: - print >>f, '/*' - for line in banner: - print >>f, ' * %s' % line - print >>f, ' */' - print >>f + for f in files: + print >> f, '/*' + for line in banner: + print >> f, ' * %s' % line + print >> f, ' */' + print >> f + with open('src/core/lib/debug/stats_data.h', 'w') as H: - # copy-paste copyright notice from this file - with open(sys.argv[0]) as my_source: - copyright = [] - for line in my_source: - if line[0] != '#': break - for line in my_source: - if line[0] == '#': - copyright.append(line) - break - for line in my_source: - if line[0] != '#': - break - copyright.append(line) - put_banner([H], [line[2:].rstrip() for line in copyright]) - - put_banner([H], ["Automatically generated by tools/codegen/core/gen_stats_data.py"]) - - print >>H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H" - print >>H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H" - print >>H - print >>H, "#include " - print >>H, "#include \"src/core/lib/iomgr/exec_ctx.h\"" - print >>H - print >>H, "#ifdef __cplusplus" - print >>H, "extern \"C\" {" - print >>H, "#endif" - print >>H - - for typename, instances in sorted(inst_map.items()): - print >>H, "typedef enum {" - for inst in instances: - print >>H, " GRPC_STATS_%s_%s," % (typename.upper(), inst.name.upper()) - print >>H, " GRPC_STATS_%s_COUNT" % (typename.upper()) - print >>H, "} grpc_stats_%ss;" % (typename.lower()) - print >>H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % ( - typename.lower(), typename.upper()) - print >>H, "extern const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT];" % ( - typename.lower(), typename.upper()) - - histo_start = [] - histo_buckets = [] - histo_bucket_boundaries = [] - - print >>H, "typedef enum {" - first_slot = 0 - for histogram in inst_map['Histogram']: - histo_start.append(first_slot) - histo_buckets.append(histogram.buckets) - print >>H, " GRPC_STATS_HISTOGRAM_%s_FIRST_SLOT = %d," % (histogram.name.upper(), first_slot) - print >>H, " GRPC_STATS_HISTOGRAM_%s_BUCKETS = %d," % (histogram.name.upper(), histogram.buckets) - first_slot += histogram.buckets - print >>H, " GRPC_STATS_HISTOGRAM_BUCKETS = %d" % first_slot - print >>H, "} grpc_stats_histogram_constants;" - - for ctr in inst_map['Counter']: - print >>H, ("#define GRPC_STATS_INC_%s() " + - "GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_%s)") % ( - ctr.name.upper(), ctr.name.upper()) - for histogram in inst_map['Histogram']: - print >>H, "#define GRPC_STATS_INC_%s(value) grpc_stats_inc_%s( (int)(value))" % ( - histogram.name.upper(), histogram.name.lower()) - print >>H, "void grpc_stats_inc_%s(int x);" % histogram.name.lower() - - for i, tbl in enumerate(static_tables): - print >>H, "extern const %s grpc_stats_table_%d[%d];" % (tbl[0], i, len(tbl[1])) - - print >>H, "extern const int grpc_stats_histo_buckets[%d];" % len(inst_map['Histogram']) - print >>H, "extern const int grpc_stats_histo_start[%d];" % len(inst_map['Histogram']) - print >>H, "extern const int *const grpc_stats_histo_bucket_boundaries[%d];" % len(inst_map['Histogram']) - print >>H, "extern void (*const grpc_stats_inc_histogram[%d])(int x);" % len(inst_map['Histogram']) - - print >>H - print >>H, "#ifdef __cplusplus" - print >>H, "}" - print >>H, "#endif" - print >>H - print >>H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */" + # copy-paste copyright notice from this file + with open(sys.argv[0]) as my_source: + copyright = [] + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + copyright.append(line) + break + for line in my_source: + if line[0] != '#': + break + copyright.append(line) + put_banner([H], [line[2:].rstrip() for line in copyright]) + + put_banner( + [H], + ["Automatically generated by tools/codegen/core/gen_stats_data.py"]) + + print >> H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H" + print >> H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H" + print >> H + print >> H, "#include " + print >> H, "#include \"src/core/lib/iomgr/exec_ctx.h\"" + print >> H + print >> H, "#ifdef __cplusplus" + print >> H, "extern \"C\" {" + print >> H, "#endif" + print >> H + + for typename, instances in sorted(inst_map.items()): + print >> H, "typedef enum {" + for inst in instances: + print >> H, " GRPC_STATS_%s_%s," % (typename.upper(), + inst.name.upper()) + print >> H, " GRPC_STATS_%s_COUNT" % (typename.upper()) + print >> H, "} grpc_stats_%ss;" % (typename.lower()) + print >> H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % ( + typename.lower(), typename.upper()) + print >> H, "extern const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT];" % ( + typename.lower(), typename.upper()) + + histo_start = [] + histo_buckets = [] + histo_bucket_boundaries = [] + + print >> H, "typedef enum {" + first_slot = 0 + for histogram in inst_map['Histogram']: + histo_start.append(first_slot) + histo_buckets.append(histogram.buckets) + print >> H, " GRPC_STATS_HISTOGRAM_%s_FIRST_SLOT = %d," % ( + histogram.name.upper(), first_slot) + print >> H, " GRPC_STATS_HISTOGRAM_%s_BUCKETS = %d," % ( + histogram.name.upper(), histogram.buckets) + first_slot += histogram.buckets + print >> H, " GRPC_STATS_HISTOGRAM_BUCKETS = %d" % first_slot + print >> H, "} grpc_stats_histogram_constants;" + + for ctr in inst_map['Counter']: + print >> H, ("#define GRPC_STATS_INC_%s() " + + "GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_%s)") % ( + ctr.name.upper(), ctr.name.upper()) + for histogram in inst_map['Histogram']: + print >> H, "#define GRPC_STATS_INC_%s(value) grpc_stats_inc_%s( (int)(value))" % ( + histogram.name.upper(), histogram.name.lower()) + print >> H, "void grpc_stats_inc_%s(int x);" % histogram.name.lower() + + for i, tbl in enumerate(static_tables): + print >> H, "extern const %s grpc_stats_table_%d[%d];" % (tbl[0], i, + len(tbl[1])) + + print >> H, "extern const int grpc_stats_histo_buckets[%d];" % len( + inst_map['Histogram']) + print >> H, "extern const int grpc_stats_histo_start[%d];" % len( + inst_map['Histogram']) + print >> H, "extern const int *const grpc_stats_histo_bucket_boundaries[%d];" % len( + inst_map['Histogram']) + print >> H, "extern void (*const grpc_stats_inc_histogram[%d])(int x);" % len( + inst_map['Histogram']) + + print >> H + print >> H, "#ifdef __cplusplus" + print >> H, "}" + print >> H, "#endif" + print >> H + print >> H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */" with open('src/core/lib/debug/stats_data.cc', 'w') as C: - # copy-paste copyright notice from this file - with open(sys.argv[0]) as my_source: - copyright = [] - for line in my_source: - if line[0] != '#': break - for line in my_source: - if line[0] == '#': - copyright.append(line) - break - for line in my_source: - if line[0] != '#': - break - copyright.append(line) - put_banner([C], [line[2:].rstrip() for line in copyright]) - - put_banner([C], ["Automatically generated by tools/codegen/core/gen_stats_data.py"]) - - print >>C, "#include \"src/core/lib/debug/stats_data.h\"" - print >>C, "#include \"src/core/lib/debug/stats.h\"" - print >>C, "#include \"src/core/lib/iomgr/exec_ctx.h\"" - print >>C, "#include " - - histo_code = [] - for histogram in inst_map['Histogram']: - code, bounds_idx = gen_bucket_code(histogram) - histo_bucket_boundaries.append(bounds_idx) - histo_code.append(code) - - for typename, instances in sorted(inst_map.items()): - print >>C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % ( - typename.lower(), typename.upper()) - for inst in instances: - print >>C, " %s," % c_str(inst.name) - print >>C, "};" - print >>C, "const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT] = {" % ( - typename.lower(), typename.upper()) - for inst in instances: - print >>C, " %s," % c_str(inst.doc) - print >>C, "};" - - for i, tbl in enumerate(static_tables): - print >>C, "const %s grpc_stats_table_%d[%d] = {%s};" % ( - tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1])) - - for histogram, code in zip(inst_map['Histogram'], histo_code): - print >>C, ("void grpc_stats_inc_%s(int value) {%s}") % ( - histogram.name.lower(), - code) - - print >>C, "const int grpc_stats_histo_buckets[%d] = {%s};" % ( - len(inst_map['Histogram']), ','.join('%s' % x for x in histo_buckets)) - print >>C, "const int grpc_stats_histo_start[%d] = {%s};" % ( - len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start)) - print >>C, "const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % ( - len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries)) - print >>C, "void (*const grpc_stats_inc_histogram[%d])(int x) = {%s};" % ( - len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram'])) + # copy-paste copyright notice from this file + with open(sys.argv[0]) as my_source: + copyright = [] + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + copyright.append(line) + break + for line in my_source: + if line[0] != '#': + break + copyright.append(line) + put_banner([C], [line[2:].rstrip() for line in copyright]) + + put_banner( + [C], + ["Automatically generated by tools/codegen/core/gen_stats_data.py"]) + + print >> C, "#include \"src/core/lib/debug/stats_data.h\"" + print >> C, "#include \"src/core/lib/debug/stats.h\"" + print >> C, "#include \"src/core/lib/iomgr/exec_ctx.h\"" + print >> C, "#include " + + histo_code = [] + for histogram in inst_map['Histogram']: + code, bounds_idx = gen_bucket_code(histogram) + histo_bucket_boundaries.append(bounds_idx) + histo_code.append(code) + + for typename, instances in sorted(inst_map.items()): + print >> C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % ( + typename.lower(), typename.upper()) + for inst in instances: + print >> C, " %s," % c_str(inst.name) + print >> C, "};" + print >> C, "const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT] = {" % ( + typename.lower(), typename.upper()) + for inst in instances: + print >> C, " %s," % c_str(inst.doc) + print >> C, "};" + + for i, tbl in enumerate(static_tables): + print >> C, "const %s grpc_stats_table_%d[%d] = {%s};" % ( + tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1])) + + for histogram, code in zip(inst_map['Histogram'], histo_code): + print >> C, ("void grpc_stats_inc_%s(int value) {%s}") % ( + histogram.name.lower(), code) + + print >> C, "const int grpc_stats_histo_buckets[%d] = {%s};" % ( + len(inst_map['Histogram']), ','.join('%s' % x for x in histo_buckets)) + print >> C, "const int grpc_stats_histo_start[%d] = {%s};" % ( + len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start)) + print >> C, "const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % ( + len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x + for x in histo_bucket_boundaries)) + print >> C, "void (*const grpc_stats_inc_histogram[%d])(int x) = {%s};" % ( + len(inst_map['Histogram']), + ','.join('grpc_stats_inc_%s' % histogram.name.lower() + for histogram in inst_map['Histogram'])) # patch qps_test bigquery schema RECORD_EXPLICIT_PERCENTILES = [50, 95, 99] with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f: - qps_schema = json.loads(f.read()) + qps_schema = json.loads(f.read()) + def FindNamed(js, name): - for el in js: - if el['name'] == name: - return el + for el in js: + if el['name'] == name: + return el + def RemoveCoreFields(js): - new_fields = [] - for field in js['fields']: - if not field['name'].startswith('core_'): - new_fields.append(field) - js['fields'] = new_fields + new_fields = [] + for field in js['fields']: + if not field['name'].startswith('core_'): + new_fields.append(field) + js['fields'] = new_fields + RemoveCoreFields(FindNamed(qps_schema, 'clientStats')) RemoveCoreFields(FindNamed(qps_schema, 'serverStats')) + def AddCoreFields(js): - for counter in inst_map['Counter']: - js['fields'].append({ - 'name': 'core_%s' % counter.name, - 'type': 'INTEGER', - 'mode': 'NULLABLE' - }) - for histogram in inst_map['Histogram']: - js['fields'].append({ - 'name': 'core_%s' % histogram.name, - 'type': 'STRING', - 'mode': 'NULLABLE' - }) - js['fields'].append({ - 'name': 'core_%s_bkts' % histogram.name, - 'type': 'STRING', - 'mode': 'NULLABLE' - }) - for pctl in RECORD_EXPLICIT_PERCENTILES: - js['fields'].append({ - 'name': 'core_%s_%dp' % (histogram.name, pctl), - 'type': 'FLOAT', - 'mode': 'NULLABLE' - }) + for counter in inst_map['Counter']: + js['fields'].append({ + 'name': 'core_%s' % counter.name, + 'type': 'INTEGER', + 'mode': 'NULLABLE' + }) + for histogram in inst_map['Histogram']: + js['fields'].append({ + 'name': 'core_%s' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + js['fields'].append({ + 'name': 'core_%s_bkts' % histogram.name, + 'type': 'STRING', + 'mode': 'NULLABLE' + }) + for pctl in RECORD_EXPLICIT_PERCENTILES: + js['fields'].append({ + 'name': 'core_%s_%dp' % (histogram.name, pctl), + 'type': 'FLOAT', + 'mode': 'NULLABLE' + }) + AddCoreFields(FindNamed(qps_schema, 'clientStats')) AddCoreFields(FindNamed(qps_schema, 'serverStats')) with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f: - f.write(json.dumps(qps_schema, indent=2, sort_keys=True)) + f.write(json.dumps(qps_schema, indent=2, sort_keys=True)) # and generate a helper script to massage scenario results into the format we'd # like to query with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P: - with open(sys.argv[0]) as my_source: - for line in my_source: - if line[0] != '#': break - for line in my_source: - if line[0] == '#': - print >>P, line.rstrip() - break - for line in my_source: - if line[0] != '#': - break - print >>P, line.rstrip() - - print >>P - print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py' - print >>P - - print >>P, 'import massage_qps_stats_helpers' - - print >>P, 'def massage_qps_stats(scenario_result):' - print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:' - print >>P, ' if "coreStats" not in stats: return' - print >>P, ' core_stats = stats["coreStats"]' - print >>P, ' del stats["coreStats"]' - for counter in inst_map['Counter']: - print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name) - for i, histogram in enumerate(inst_map['Histogram']): - print >>P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name - print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name - print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name - for pctl in RECORD_EXPLICIT_PERCENTILES: - print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % ( - histogram.name, pctl, pctl) + with open(sys.argv[0]) as my_source: + for line in my_source: + if line[0] != '#': break + for line in my_source: + if line[0] == '#': + print >> P, line.rstrip() + break + for line in my_source: + if line[0] != '#': + break + print >> P, line.rstrip() + + print >> P + print >> P, '# Autogenerated by tools/codegen/core/gen_stats_data.py' + print >> P + + print >> P, 'import massage_qps_stats_helpers' + + print >> P, 'def massage_qps_stats(scenario_result):' + print >> P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:' + print >> P, ' if "coreStats" not in stats: return' + print >> P, ' core_stats = stats["coreStats"]' + print >> P, ' del stats["coreStats"]' + for counter in inst_map['Counter']: + print >> P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % ( + counter.name, counter.name) + for i, histogram in enumerate(inst_map['Histogram']): + print >> P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name + print >> P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name + print >> P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name + for pctl in RECORD_EXPLICIT_PERCENTILES: + print >> P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % ( + histogram.name, pctl, pctl) with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: - columns = [] - for counter in inst_map['Counter']: - columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) - print >>S, ',\n'.join('%s:%s' % x for x in columns) - + columns = [] + for counter in inst_map['Counter']: + columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) + print >> S, ',\n'.join('%s:%s' % x for x in columns) diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh index 7024168b2d8..b784f60e4dd 100755 --- a/tools/distrib/yapf_code.sh +++ b/tools/distrib/yapf_code.sh @@ -21,6 +21,7 @@ cd "$(dirname "${0}")/../.." DIRS=( 'src/python' 'tools/buildgen' + 'tools/codegen' ) EXCLUSIONS=( 'grpcio/grpc_*.py'