Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

yapf tools/codegen

Browse Source
pull/13719/head
ncteisen 7 years ago
parent 26d70b1653
commit 5898847ddf
6 changed files with 777 additions and 720 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 154
      tools/codegen/core/gen_header_frame.py
  2. 74
      tools/codegen/core/gen_server_registered_method_bad_client_test_body.py
  3. 189
      tools/codegen/core/gen_settings_ids.py
  4. 374
      tools/codegen/core/gen_static_metadata.py
  5. 705
      tools/codegen/core/gen_stats_data.py
  6. 1
      tools/distrib/yapf_code.sh

154
tools/codegen/core/gen_header_frame.py
Unescape View File

@ -13,7 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Read from stdin a set of colon separated http headers:
:path: /foo/bar
content-type: application/grpc
@ -24,109 +23,118 @@ import json
import sys
import argparse
def append_never_indexed(payload_line, n, count, key, value):
payload_line.append(0x10)
assert(len(key) <= 126)
payload_line.append(len(key))
payload_line.extend(ord(c) for c in key)
assert(len(value) <= 126)
payload_line.append(len(value))
payload_line.extend(ord(c) for c in value)
payload_line.append(0x10)
assert (len(key) <= 126)
payload_line.append(len(key))
payload_line.extend(ord(c) for c in key)
assert (len(value) <= 126)
payload_line.append(len(value))
payload_line.extend(ord(c) for c in value)
def append_inc_indexed(payload_line, n, count, key, value):
payload_line.append(0x40)
assert(len(key) <= 126)
payload_line.append(len(key))
payload_line.extend(ord(c) for c in key)
assert(len(value) <= 126)
payload_line.append(len(value))
payload_line.extend(ord(c) for c in value)
payload_line.append(0x40)
assert (len(key) <= 126)
payload_line.append(len(key))
payload_line.extend(ord(c) for c in key)
assert (len(value) <= 126)
payload_line.append(len(value))
payload_line.extend(ord(c) for c in value)
def append_pre_indexed(payload_line, n, count, key, value):
payload_line.append(0x80 + 61 + count - n)
payload_line.append(0x80 + 61 + count - n)
_COMPRESSORS = {
'never': append_never_indexed,
'inc': append_inc_indexed,
'pre': append_pre_indexed,
'never': append_never_indexed,
'inc': append_inc_indexed,
'pre': append_pre_indexed,
}
argp = argparse.ArgumentParser('Generate header frames')
argp.add_argument('--set_end_stream', default=False, action='store_const', const=True)
argp.add_argument('--no_framing', default=False, action='store_const', const=True)
argp.add_argument('--compression', choices=sorted(_COMPRESSORS.keys()), default='never')
argp.add_argument(
'--set_end_stream', default=False, action='store_const', const=True)
argp.add_argument(
'--no_framing', default=False, action='store_const', const=True)
argp.add_argument(
'--compression', choices=sorted(_COMPRESSORS.keys()), default='never')
argp.add_argument('--hex', default=False, action='store_const', const=True)
args = argp.parse_args()
# parse input, fill in vals
vals = []
for line in sys.stdin:
line = line.strip()
if line == '': continue
if line[0] == '#': continue
key_tail, value = line[1:].split(':')
key = (line[0] + key_tail).strip()
value = value.strip()
vals.append((key, value))
line = line.strip()
if line == '': continue
if line[0] == '#': continue
key_tail, value = line[1:].split(':')
key = (line[0] + key_tail).strip()
value = value.strip()
vals.append((key, value))
# generate frame payload binary data
payload_bytes = []
if not args.no_framing:
payload_bytes.append([]) # reserve space for header
payload_bytes.append([]) # reserve space for header
payload_len = 0
n = 0
for key, value in vals:
payload_line = []
_COMPRESSORS[args.compression](payload_line, n, len(vals), key, value)
n += 1
payload_len += len(payload_line)
payload_bytes.append(payload_line)
payload_line = []
_COMPRESSORS[args.compression](payload_line, n, len(vals), key, value)
n += 1
payload_len += len(payload_line)
payload_bytes.append(payload_line)
# fill in header
if not args.no_framing:
flags = 0x04 # END_HEADERS
if args.set_end_stream:
flags |= 0x01 # END_STREAM
payload_bytes[0].extend([
(payload_len >> 16) & 0xff,
(payload_len >> 8) & 0xff,
(payload_len) & 0xff,
# header frame
0x01,
# flags
flags,
# stream id
0x00,
0x00,
0x00,
0x01
])
flags = 0x04 # END_HEADERS
if args.set_end_stream:
flags |= 0x01 # END_STREAM
payload_bytes[0].extend([
(payload_len >> 16) & 0xff,
(payload_len >> 8) & 0xff,
(payload_len) & 0xff,
# header frame
0x01,
# flags
flags,
# stream id
0x00,
0x00,
0x00,
0x01
])
hex_bytes = [ord(c) for c in "abcdefABCDEF0123456789"]
def esc_c(line):
out = "\""
last_was_hex = False
for c in line:
if 32 <= c < 127:
if c in hex_bytes and last_was_hex:
out += "\"\""
if c != ord('"'):
out += chr(c)
else:
out += "\\\""
last_was_hex = False
else:
out += "\\x%02x" % c
last_was_hex = True
return out + "\""
out = "\""
last_was_hex = False
for c in line:
if 32 <= c < 127:
if c in hex_bytes and last_was_hex:
out += "\"\""
if c != ord('"'):
out += chr(c)
else:
out += "\\\""
last_was_hex = False
else:
out += "\\x%02x" % c
last_was_hex = True
return out + "\""
# dump bytes
if args.hex:
all_bytes = []
for line in payload_bytes:
all_bytes.extend(line)
print '{%s}' % ', '.join('0x%02x' % c for c in all_bytes)
all_bytes = []
for line in payload_bytes:
all_bytes.extend(line)
print '{%s}' % ', '.join('0x%02x' % c for c in all_bytes)
else:
for line in payload_bytes:
print esc_c(line)
for line in payload_bytes:
print esc_c(line)

74
tools/codegen/core/gen_server_registered_method_bad_client_test_body.py
Unescape View File

@ -14,48 +14,42 @@
# See the License for the specific language governing permissions and
# limitations under the License.
def esc_c(line):
out = "\""
last_was_hex = False
for c in line:
if 32 <= c < 127:
if c in hex_bytes and last_was_hex:
out += "\"\""
if c != ord('"'):
out += chr(c)
else:
out += "\\\""
last_was_hex = False
else:
out += "\\x%02x" % c
last_was_hex = True
return out + "\""
out = "\""
last_was_hex = False
for c in line:
if 32 <= c < 127:
if c in hex_bytes and last_was_hex:
out += "\"\""
if c != ord('"'):
out += chr(c)
else:
out += "\\\""
last_was_hex = False
else:
out += "\\x%02x" % c
last_was_hex = True
return out + "\""
done = set()
for message_length in range(0, 3):
for send_message_length in range(0, message_length + 1):
payload = [
0,
(message_length >> 24) & 0xff,
(message_length >> 16) & 0xff,
(message_length >> 8) & 0xff,
(message_length) & 0xff
] + send_message_length * [0]
for frame_length in range(0, len(payload) + 1):
is_end = frame_length == len(payload) and send_message_length == message_length
frame = [
(frame_length >> 16) & 0xff,
(frame_length >> 8) & 0xff,
(frame_length) & 0xff,
0,
1 if is_end else 0,
0, 0, 0, 1
] + payload[0:frame_length]
text = esc_c(frame)
if text not in done:
print 'GRPC_RUN_BAD_CLIENT_TEST(verifier_%s, PFX_STR %s, %s);' % (
'succeeds' if is_end else 'fails',
text,
'0' if is_end else 'GRPC_BAD_CLIENT_DISCONNECT')
done.add(text)
for send_message_length in range(0, message_length + 1):
payload = [
0, (message_length >> 24) & 0xff, (message_length >> 16) & 0xff,
(message_length >> 8) & 0xff, (message_length) & 0xff
] + send_message_length * [0]
for frame_length in range(0, len(payload) + 1):
is_end = frame_length == len(
payload) and send_message_length == message_length
frame = [(frame_length >> 16) & 0xff, (frame_length >> 8) & 0xff,
(frame_length) & 0xff, 0, 1
if is_end else 0, 0, 0, 0, 1] + payload[0:frame_length]
text = esc_c(frame)
if text not in done:
print 'GRPC_RUN_BAD_CLIENT_TEST(verifier_%s, PFX_STR %s, %s);' % (
'succeeds' if is_end else 'fails', text, '0'
if is_end else 'GRPC_BAD_CLIENT_DISCONNECT')
done.add(text)

189
tools/codegen/core/gen_settings_ids.py
Unescape View File

@ -24,92 +24,114 @@ Setting = collections.namedtuple('Setting', 'id default min max on_error')
OnError = collections.namedtuple('OnError', 'behavior code')
clamp_invalid_value = OnError('CLAMP_INVALID_VALUE', 'PROTOCOL_ERROR')
disconnect_on_invalid_value = lambda e: OnError('DISCONNECT_ON_INVALID_VALUE', e)
DecoratedSetting = collections.namedtuple('DecoratedSetting', 'enum name setting')
DecoratedSetting = collections.namedtuple('DecoratedSetting',
'enum name setting')
_SETTINGS = {
'HEADER_TABLE_SIZE': Setting(1, 4096, 0, 0xffffffff, clamp_invalid_value),
'ENABLE_PUSH': Setting(2, 1, 0, 1, disconnect_on_invalid_value('PROTOCOL_ERROR')),
'MAX_CONCURRENT_STREAMS': Setting(3, 0xffffffff, 0, 0xffffffff, disconnect_on_invalid_value('PROTOCOL_ERROR')),
'INITIAL_WINDOW_SIZE': Setting(4, 65535, 0, 0x7fffffff, disconnect_on_invalid_value('FLOW_CONTROL_ERROR')),
'MAX_FRAME_SIZE': Setting(5, 16384, 16384, 16777215, disconnect_on_invalid_value('PROTOCOL_ERROR')),
'MAX_HEADER_LIST_SIZE': Setting(6, _MAX_HEADER_LIST_SIZE, 0, _MAX_HEADER_LIST_SIZE, clamp_invalid_value),
'GRPC_ALLOW_TRUE_BINARY_METADATA': Setting(0xfe03, 0, 0, 1, clamp_invalid_value),
'HEADER_TABLE_SIZE':
Setting(1, 4096, 0, 0xffffffff, clamp_invalid_value),
'ENABLE_PUSH':
Setting(2, 1, 0, 1, disconnect_on_invalid_value('PROTOCOL_ERROR')),
'MAX_CONCURRENT_STREAMS':
Setting(3, 0xffffffff, 0, 0xffffffff,
disconnect_on_invalid_value('PROTOCOL_ERROR')),
'INITIAL_WINDOW_SIZE':
Setting(4, 65535, 0, 0x7fffffff,
disconnect_on_invalid_value('FLOW_CONTROL_ERROR')),
'MAX_FRAME_SIZE':
Setting(5, 16384, 16384, 16777215,
disconnect_on_invalid_value('PROTOCOL_ERROR')),
'MAX_HEADER_LIST_SIZE':
Setting(6, _MAX_HEADER_LIST_SIZE, 0, _MAX_HEADER_LIST_SIZE,
clamp_invalid_value),
'GRPC_ALLOW_TRUE_BINARY_METADATA':
Setting(0xfe03, 0, 0, 1, clamp_invalid_value),
}
H = open('src/core/ext/transport/chttp2/transport/http2_settings.h', 'w')
C = open('src/core/ext/transport/chttp2/transport/http2_settings.c', 'w')
# utility: print a big comment block into a set of files
def put_banner(files, banner):
for f in files:
print >>f, '/*'
for line in banner:
print >>f, ' * %s' % line
print >>f, ' */'
print >>f
for f in files:
print >> f, '/*'
for line in banner:
print >> f, ' * %s' % line
print >> f, ' */'
print >> f
# copy-paste copyright notice from this file
with open(sys.argv[0]) as my_source:
copyright = []
for line in my_source:
if line[0] != '#': break
for line in my_source:
if line[0] == '#':
copyright.append(line)
break
for line in my_source:
if line[0] != '#':
break
copyright.append(line)
put_banner([H,C], [line[2:].rstrip() for line in copyright])
put_banner([H,C], ["Automatically generated by tools/codegen/core/gen_settings_ids.py"])
print >>H, "#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H"
print >>H, "#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H"
print >>H
print >>H, "#include <stdint.h>"
print >>H, "#include <stdbool.h>"
print >>H
print >>C, "#include \"src/core/ext/transport/chttp2/transport/http2_settings.h\""
print >>C
print >>C, "#include <grpc/support/useful.h>"
print >>C, "#include \"src/core/lib/transport/http2_errors.h\""
print >>C
copyright = []
for line in my_source:
if line[0] != '#': break
for line in my_source:
if line[0] == '#':
copyright.append(line)
break
for line in my_source:
if line[0] != '#':
break
copyright.append(line)
put_banner([H, C], [line[2:].rstrip() for line in copyright])
put_banner(
[H, C],
["Automatically generated by tools/codegen/core/gen_settings_ids.py"])
print >> H, "#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H"
print >> H, "#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H"
print >> H
print >> H, "#include <stdint.h>"
print >> H, "#include <stdbool.h>"
print >> H
print >> C, "#include \"src/core/ext/transport/chttp2/transport/http2_settings.h\""
print >> C
print >> C, "#include <grpc/support/useful.h>"
print >> C, "#include \"src/core/lib/transport/http2_errors.h\""
print >> C
p = perfection.hash_parameters(sorted(x.id for x in _SETTINGS.values()))
print p
def hash(i):
i += p.offset
x = i % p.t
y = i / p.t
return x + p.r[y]
i += p.offset
x = i % p.t
y = i / p.t
return x + p.r[y]
decorated_settings = [DecoratedSetting(hash(setting.id), name, setting)
for name, setting in _SETTINGS.iteritems()]
decorated_settings = [
DecoratedSetting(hash(setting.id), name, setting)
for name, setting in _SETTINGS.iteritems()
]
print >>H, 'typedef enum {'
print >> H, 'typedef enum {'
for decorated_setting in sorted(decorated_settings):
print >>H, ' GRPC_CHTTP2_SETTINGS_%s = %d, /* wire id %d */' % (
decorated_setting.name, decorated_setting.enum, decorated_setting.setting.id)
print >>H, '} grpc_chttp2_setting_id;'
print >>H
print >>H, '#define GRPC_CHTTP2_NUM_SETTINGS %d' % (max(x.enum for x in decorated_settings) + 1)
print >>H, 'extern const uint16_t grpc_setting_id_to_wire_id[];'
print >>C, 'const uint16_t grpc_setting_id_to_wire_id[] = {%s};' % ','.join(
print >> H, ' GRPC_CHTTP2_SETTINGS_%s = %d, /* wire id %d */' % (
decorated_setting.name, decorated_setting.enum,
decorated_setting.setting.id)
print >> H, '} grpc_chttp2_setting_id;'
print >> H
print >> H, '#define GRPC_CHTTP2_NUM_SETTINGS %d' % (
max(x.enum for x in decorated_settings) + 1)
print >> H, 'extern const uint16_t grpc_setting_id_to_wire_id[];'
print >> C, 'const uint16_t grpc_setting_id_to_wire_id[] = {%s};' % ','.join(
'%d' % s for s in p.slots)
print >>H
print >>H, "bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out);"
print >> H
print >> H, "bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out);"
cgargs = {
'r': ','.join('%d' % (r if r is not None else 0) for r in p.r),
't': p.t,
'offset': abs(p.offset),
'offset_sign': '+' if p.offset > 0 else '-'
}
print >>C, """
'r': ','.join('%d' % (r if r is not None else 0) for r in p.r),
't': p.t,
'offset': abs(p.offset),
'offset_sign': '+' if p.offset > 0 else '-'
}
print >> C, """
bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out) {
uint32_t i = wire_id %(offset_sign)s %(offset)d;
uint32_t x = i %% %(t)d;
@ -118,17 +140,17 @@ bool grpc_wire_id_to_setting_id(uint32_t wire_id, grpc_chttp2_setting_id *out) {
switch (y) {
""" % cgargs
for i, r in enumerate(p.r):
if not r: continue
if r < 0: print >>C, 'case %d: h -= %d; break;' % (i, -r)
else: print >>C, 'case %d: h += %d; break;' % (i, r)
print >>C, """
if not r: continue
if r < 0: print >> C, 'case %d: h -= %d; break;' % (i, -r)
else: print >> C, 'case %d: h += %d; break;' % (i, r)
print >> C, """
}
*out = (grpc_chttp2_setting_id)h;
return h < GPR_ARRAY_SIZE(grpc_setting_id_to_wire_id) && grpc_setting_id_to_wire_id[h] == wire_id;
}
""" % cgargs
print >>H, """
print >> H, """
typedef enum {
GRPC_CHTTP2_CLAMP_INVALID_VALUE,
GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE
@ -145,25 +167,22 @@ typedef struct {
extern const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS];
"""
print >>C, "const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS] = {"
print >> C, "const grpc_chttp2_setting_parameters grpc_chttp2_settings_parameters[GRPC_CHTTP2_NUM_SETTINGS] = {"
i = 0
for decorated_setting in sorted(decorated_settings):
while i < decorated_setting.enum:
print >>C, "{NULL, 0, 0, 0, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_HTTP2_PROTOCOL_ERROR},"
while i < decorated_setting.enum:
print >> C, "{NULL, 0, 0, 0, GRPC_CHTTP2_DISCONNECT_ON_INVALID_VALUE, GRPC_HTTP2_PROTOCOL_ERROR},"
i += 1
print >> C, "{\"%s\", %du, %du, %du, GRPC_CHTTP2_%s, GRPC_HTTP2_%s}," % (
decorated_setting.name, decorated_setting.setting.default,
decorated_setting.setting.min, decorated_setting.setting.max,
decorated_setting.setting.on_error.behavior,
decorated_setting.setting.on_error.code,)
i += 1
print >>C, "{\"%s\", %du, %du, %du, GRPC_CHTTP2_%s, GRPC_HTTP2_%s}," % (
decorated_setting.name,
decorated_setting.setting.default,
decorated_setting.setting.min,
decorated_setting.setting.max,
decorated_setting.setting.on_error.behavior,
decorated_setting.setting.on_error.code,
)
i += 1
print >>C, "};"
print >>H
print >>H, "#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H */"
print >> C, "};"
print >> H
print >> H, "#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HTTP2_SETTINGS_H */"
H.close()
C.close()

374
tools/codegen/core/gen_static_metadata.py
Unescape View File

@ -172,65 +172,66 @@ STREAM_COMPRESSION_ALGORITHMS = [
'gzip',
]
# utility: mangle the name of a config
def mangle(elem, name=None):
xl = {
'-': '_',
':': '',
'/': 'slash',
'.': 'dot',
',': 'comma',
' ': '_',
}
xl = {
'-': '_',
':': '',
'/': 'slash',
'.': 'dot',
',': 'comma',
' ': '_',
}
def m0(x):
if not x:
return 'empty'
r = ''
for c in x:
put = xl.get(c, c.lower())
if not put:
continue
last_is_underscore = r[-1] == '_' if r else True
if last_is_underscore and put == '_':
continue
elif len(put) > 1:
if not last_is_underscore:
r += '_'
r += put
r += '_'
else:
r += put
if r[-1] == '_':
r = r[:-1]
return r
def n(default, name=name):
if name is None:
return 'grpc_%s_' % default
if name == '':
return ''
return 'grpc_%s_' % name
if isinstance(elem, tuple):
return '%s%s_%s' % (n('mdelem'), m0(elem[0]), m0(elem[1]))
else:
return '%s%s' % (n('mdstr'), m0(elem))
def m0(x):
if not x:
return 'empty'
r = ''
for c in x:
put = xl.get(c, c.lower())
if not put:
continue
last_is_underscore = r[-1] == '_' if r else True
if last_is_underscore and put == '_':
continue
elif len(put) > 1:
if not last_is_underscore:
r += '_'
r += put
r += '_'
else:
r += put
if r[-1] == '_':
r = r[:-1]
return r
def n(default, name=name):
if name is None:
return 'grpc_%s_' % default
if name == '':
return ''
return 'grpc_%s_' % name
if isinstance(elem, tuple):
return '%s%s_%s' % (n('mdelem'), m0(elem[0]), m0(elem[1]))
else:
return '%s%s' % (n('mdstr'), m0(elem))
# utility: generate some hash value for a string
def fake_hash(elem):
return hashlib.md5(elem).hexdigest()[0:8]
return hashlib.md5(elem).hexdigest()[0:8]
# utility: print a big comment block into a set of files
def put_banner(files, banner):
for f in files:
print >> f, '/*'
for line in banner:
print >> f, ' * %s' % line
print >> f, ' */'
print >> f
for f in files:
print >> f, '/*'
for line in banner:
print >> f, ' * %s' % line
print >> f, ' */'
print >> f
# build a list of all the strings we need
@ -240,43 +241,43 @@ static_userdata = {}
# put metadata batch callouts first, to make the check of if a static metadata
# string is a callout trivial
for elem, _ in METADATA_BATCH_CALLOUTS:
if elem not in all_strs:
all_strs.append(elem)
for elem in CONFIG:
if isinstance(elem, tuple):
if elem[0] not in all_strs:
all_strs.append(elem[0])
if elem[1] not in all_strs:
all_strs.append(elem[1])
if elem not in all_elems:
all_elems.append(elem)
else:
if elem not in all_strs:
all_strs.append(elem)
all_strs.append(elem)
for elem in CONFIG:
if isinstance(elem, tuple):
if elem[0] not in all_strs:
all_strs.append(elem[0])
if elem[1] not in all_strs:
all_strs.append(elem[1])
if elem not in all_elems:
all_elems.append(elem)
else:
if elem not in all_strs:
all_strs.append(elem)
compression_elems = []
for mask in range(1, 1 << len(COMPRESSION_ALGORITHMS)):
val = ','.join(COMPRESSION_ALGORITHMS[alg]
for alg in range(0, len(COMPRESSION_ALGORITHMS))
if (1 << alg) & mask)
elem = ('grpc-accept-encoding', val)
if val not in all_strs:
all_strs.append(val)
if elem not in all_elems:
all_elems.append(elem)
compression_elems.append(elem)
static_userdata[elem] = 1 + (mask | 1)
val = ','.join(COMPRESSION_ALGORITHMS[alg]
for alg in range(0, len(COMPRESSION_ALGORITHMS))
if (1 << alg) & mask)
elem = ('grpc-accept-encoding', val)
if val not in all_strs:
all_strs.append(val)
if elem not in all_elems:
all_elems.append(elem)
compression_elems.append(elem)
static_userdata[elem] = 1 + (mask | 1)
stream_compression_elems = []
for mask in range(1, 1 << len(STREAM_COMPRESSION_ALGORITHMS)):
val = ','.join(STREAM_COMPRESSION_ALGORITHMS[alg]
for alg in range(0, len(STREAM_COMPRESSION_ALGORITHMS))
if (1 << alg) & mask)
elem = ('accept-encoding', val)
if val not in all_strs:
all_strs.append(val)
if elem not in all_elems:
all_elems.append(elem)
stream_compression_elems.append(elem)
static_userdata[elem] = 1 + (mask | 1)
val = ','.join(STREAM_COMPRESSION_ALGORITHMS[alg]
for alg in range(0, len(STREAM_COMPRESSION_ALGORITHMS))
if (1 << alg) & mask)
elem = ('accept-encoding', val)
if val not in all_strs:
all_strs.append(val)
if elem not in all_elems:
all_elems.append(elem)
stream_compression_elems.append(elem)
static_userdata[elem] = 1 + (mask | 1)
# output configuration
args = sys.argv[1:]
@ -284,62 +285,62 @@ H = None
C = None
D = None
if args:
if 'header' in args:
H = sys.stdout
else:
H = open('/dev/null', 'w')
if 'source' in args:
C = sys.stdout
else:
C = open('/dev/null', 'w')
if 'dictionary' in args:
D = sys.stdout
else:
D = open('/dev/null', 'w')
if 'header' in args:
H = sys.stdout
else:
H = open('/dev/null', 'w')
if 'source' in args:
C = sys.stdout
else:
C = open('/dev/null', 'w')
if 'dictionary' in args:
D = sys.stdout
else:
D = open('/dev/null', 'w')
else:
H = open(
os.path.join(
os.path.dirname(sys.argv[0]),
'../../../src/core/lib/transport/static_metadata.h'), 'w')
C = open(
os.path.join(
os.path.dirname(sys.argv[0]),
'../../../src/core/lib/transport/static_metadata.cc'), 'w')
D = open(
os.path.join(
os.path.dirname(sys.argv[0]),
'../../../test/core/end2end/fuzzers/hpack.dictionary'), 'w')
H = open(
os.path.join(
os.path.dirname(sys.argv[0]),
'../../../src/core/lib/transport/static_metadata.h'), 'w')
C = open(
os.path.join(
os.path.dirname(sys.argv[0]),
'../../../src/core/lib/transport/static_metadata.cc'), 'w')
D = open(
os.path.join(
os.path.dirname(sys.argv[0]),
'../../../test/core/end2end/fuzzers/hpack.dictionary'), 'w')
# copy-paste copyright notice from this file
with open(sys.argv[0]) as my_source:
copyright = []
for line in my_source:
if line[0] != '#':
break
for line in my_source:
if line[0] == '#':
copyright.append(line)
break
for line in my_source:
if line[0] != '#':
break
copyright.append(line)
put_banner([H, C], [line[2:].rstrip() for line in copyright])
copyright = []
for line in my_source:
if line[0] != '#':
break
for line in my_source:
if line[0] == '#':
copyright.append(line)
break
for line in my_source:
if line[0] != '#':
break
copyright.append(line)
put_banner([H, C], [line[2:].rstrip() for line in copyright])
hex_bytes = [ord(c) for c in 'abcdefABCDEF0123456789']
def esc_dict(line):
out = "\""
for c in line:
if 32 <= c < 127:
if c != ord('"'):
out += chr(c)
else:
out += "\\\""
else:
out += '\\x%02X' % c
return out + "\""
out = "\""
for c in line:
if 32 <= c < 127:
if c != ord('"'):
out += chr(c)
else:
out += "\\\""
else:
out += '\\x%02X' % c
return out + "\""
put_banner([H, C], """WARNING: Auto-generated code.
@ -369,27 +370,26 @@ print >> C
str_ofs = 0
id2strofs = {}
for i, elem in enumerate(all_strs):
id2strofs[i] = str_ofs
str_ofs += len(elem)
id2strofs[i] = str_ofs
str_ofs += len(elem)
def slice_def(i):
return ('{&grpc_static_metadata_refcounts[%d],'
' {{g_bytes+%d, %d}}}') % (
i, id2strofs[i], len(all_strs[i]))
return ('{&grpc_static_metadata_refcounts[%d],'
' {{g_bytes+%d, %d}}}') % (i, id2strofs[i], len(all_strs[i]))
# validate configuration
for elem, _ in METADATA_BATCH_CALLOUTS:
assert elem in all_strs
assert elem in all_strs
print >> H, '#define GRPC_STATIC_MDSTR_COUNT %d' % len(all_strs)
print >> H, ('extern const grpc_slice '
'grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT];')
for i, elem in enumerate(all_strs):
print >> H, '/* "%s" */' % elem
print >> H, '#define %s (grpc_static_slice_table[%d])' % (
mangle(elem).upper(), i)
print >> H, '/* "%s" */' % elem
print >> H, '#define %s (grpc_static_slice_table[%d])' % (
mangle(elem).upper(), i)
print >> H
print >> C, 'static uint8_t g_bytes[] = {%s};' % (
','.join('%d' % ord(c) for c in ''.join(all_strs)))
@ -411,7 +411,7 @@ print >> H, ('extern grpc_slice_refcount '
print >> C, ('grpc_slice_refcount '
'grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = {')
for i, elem in enumerate(all_strs):
print >> C, ' {&grpc_static_metadata_vtable, &static_sub_refcnt},'
print >> C, ' {&grpc_static_metadata_vtable, &static_sub_refcnt},'
print >> C, '};'
print >> C
print >> H, '#define GRPC_IS_STATIC_METADATA_STRING(slice) \\'
@ -421,7 +421,7 @@ print >> H
print >> C, ('const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]'
' = {')
for i, elem in enumerate(all_strs):
print >> C, slice_def(i) + ','
print >> C, slice_def(i) + ','
print >> C, '};'
print >> C
print >> H, '#define GRPC_STATIC_METADATA_INDEX(static_slice) \\'
@ -431,10 +431,10 @@ print >> H
print >> D, '# hpack fuzzing dictionary'
for i, elem in enumerate(all_strs):
print >> D, '%s' % (esc_dict([len(elem)] + [ord(c) for c in elem]))
print >> D, '%s' % (esc_dict([len(elem)] + [ord(c) for c in elem]))
for i, elem in enumerate(all_elems):
print >> D, '%s' % (esc_dict([0, len(elem[0])] + [ord(c) for c in elem[0]] +
[len(elem[1])] + [ord(c) for c in elem[1]]))
print >> D, '%s' % (esc_dict([0, len(elem[0])] + [ord(c) for c in elem[0]] +
[len(elem[1])] + [ord(c) for c in elem[1]]))
print >> H, '#define GRPC_STATIC_MDELEM_COUNT %d' % len(all_elems)
print >> H, ('extern grpc_mdelem_data '
@ -442,10 +442,9 @@ print >> H, ('extern grpc_mdelem_data '
print >> H, ('extern uintptr_t '
'grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];')
for i, elem in enumerate(all_elems):
print >> H, '/* "%s": "%s" */' % elem
print >> H, ('#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d], '
'GRPC_MDELEM_STORAGE_STATIC))') % (
mangle(elem).upper(), i)
print >> H, '/* "%s": "%s" */' % elem
print >> H, ('#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d], '
'GRPC_MDELEM_STORAGE_STATIC))') % (mangle(elem).upper(), i)
print >> H
print >> C, ('uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] '
'= {')
@ -456,42 +455,38 @@ print >> C
def str_idx(s):
for i, s2 in enumerate(all_strs):
if s == s2:
return i
for i, s2 in enumerate(all_strs):
if s == s2:
return i
def md_idx(m):
for i, m2 in enumerate(all_elems):
if m == m2:
return i
for i, m2 in enumerate(all_elems):
if m == m2:
return i
def offset_trials(mink):
yield 0
for i in range(1, 100):
for mul in [-1, 1]:
yield mul * i
yield 0
for i in range(1, 100):
for mul in [-1, 1]:
yield mul * i
def perfect_hash(keys, name):
p = perfection.hash_parameters(keys)
def f(i, p=p):
i += p.offset
x = i % p.t
y = i / p.t
return x + p.r[y]
return {
'PHASHRANGE':
p.t - 1 + max(p.r),
'PHASHNKEYS':
len(p.slots),
'pyfunc':
f,
'code':
"""
p = perfection.hash_parameters(keys)
def f(i, p=p):
i += p.offset
x = i % p.t
y = i / p.t
return x + p.r[y]
return {
'PHASHRANGE': p.t - 1 + max(p.r),
'PHASHNKEYS': len(p.slots),
'pyfunc': f,
'code': """
static const int8_t %(name)s_r[] = {%(r)s};
static uint32_t %(name)s_phash(uint32_t i) {
i %(offset_sign)s= %(offset)d;
@ -505,13 +500,13 @@ static uint32_t %(name)s_phash(uint32_t i) {
return h;
}
""" % {
'name': name,
'r': ','.join('%d' % (r if r is not None else 0) for r in p.r),
't': p.t,
'offset': abs(p.offset),
'offset_sign': '+' if p.offset > 0 else '-'
'name': name,
'r': ','.join('%d' % (r if r is not None else 0) for r in p.r),
't': p.t,
'offset': abs(p.offset),
'offset_sign': '+' if p.offset > 0 else '-'
}
}
}
elem_keys = [
@ -523,14 +518,14 @@ print >> C, elem_hash['code']
keys = [0] * int(elem_hash['PHASHRANGE'])
idxs = [255] * int(elem_hash['PHASHNKEYS'])
for i, k in enumerate(elem_keys):
h = elem_hash['pyfunc'](k)
assert keys[h] == 0
keys[h] = k
idxs[h] = i
h = elem_hash['pyfunc'](k)
assert keys[h] == 0
keys[h] = k
idxs[h] = i
print >> C, 'static const uint16_t elem_keys[] = {%s};' % ','.join(
'%d' % k for k in keys)
print >> C, 'static const uint8_t elem_idxs[] = {%s};' % ','.join(
'%d' % i for i in idxs)
print >> C, 'static const uint8_t elem_idxs[] = {%s};' % ','.join('%d' % i
for i in idxs)
print >> C
print >> H, 'grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b);'
@ -544,12 +539,12 @@ print >> C
print >> C, 'grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = {'
for a, b in all_elems:
print >> C, '{%s,%s},' % (slice_def(str_idx(a)), slice_def(str_idx(b)))
print >> C, '{%s,%s},' % (slice_def(str_idx(a)), slice_def(str_idx(b)))
print >> C, '};'
print >> H, 'typedef enum {'
for elem, _ in METADATA_BATCH_CALLOUTS:
print >> H, ' %s,' % mangle(elem, 'batch').upper()
print >> H, ' %s,' % mangle(elem, 'batch').upper()
print >> H, ' GRPC_BATCH_CALLOUTS_COUNT'
print >> H, '} grpc_metadata_batch_callouts_index;'
print >> H
@ -557,7 +552,7 @@ print >> H, 'typedef union {'
print >> H, ' struct grpc_linked_mdelem *array[GRPC_BATCH_CALLOUTS_COUNT];'
print >> H, ' struct {'
for elem, _ in METADATA_BATCH_CALLOUTS:
print >> H, ' struct grpc_linked_mdelem *%s;' % mangle(elem, '').lower()
print >> H, ' struct grpc_linked_mdelem *%s;' % mangle(elem, '').lower()
print >> H, ' } named;'
print >> H, '} grpc_metadata_batch_callouts;'
print >> H
@ -569,7 +564,7 @@ print >> H, ('extern bool grpc_static_callout_is_default['
print >> H
print >> C, 'bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = {'
for elem, is_default in METADATA_BATCH_CALLOUTS:
print >> C, ' %s, // %s' % (str(is_default).lower(), elem)
print >> C, ' %s, // %s' % (str(is_default).lower(), elem)
print >> C, '};'
print >> C
@ -588,7 +583,8 @@ print >> H, 'extern const uint8_t grpc_static_accept_stream_encoding_metadata[%d
1 << len(STREAM_COMPRESSION_ALGORITHMS))
print >> C, 'const uint8_t grpc_static_accept_stream_encoding_metadata[%d] = {' % (
1 << len(STREAM_COMPRESSION_ALGORITHMS))
print >> C, '0,%s' % ','.join('%d' % md_idx(elem) for elem in stream_compression_elems)
print >> C, '0,%s' % ','.join('%d' % md_idx(elem)
for elem in stream_compression_elems)
print >> C, '};'
print >> H, '#define GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS(algs) (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[grpc_static_accept_stream_encoding_metadata[(algs)]], GRPC_MDELEM_STORAGE_STATIC))'

705
tools/codegen/core/gen_stats_data.py
Unescape View File

@ -22,397 +22,436 @@ import yaml
import json
with open('src/core/lib/debug/stats_data.yaml') as f:
attrs = yaml.load(f.read())
attrs = yaml.load(f.read())
REQUIRED_FIELDS = ['name', 'doc']
def make_type(name, fields):
return (collections.namedtuple(name, ' '.join(list(set(REQUIRED_FIELDS + fields)))), [])
return (collections.namedtuple(
name, ' '.join(list(set(REQUIRED_FIELDS + fields)))), [])
def c_str(s, encoding='ascii'):
if isinstance(s, unicode):
s = s.encode(encoding)
result = ''
for c in s:
if not (32 <= ord(c) < 127) or c in ('\\', '"'):
result += '\\%03o' % ord(c)
else:
result += c
return '"' + result + '"'
types = (
make_type('Counter', []),
make_type('Histogram', ['max', 'buckets']),
)
if isinstance(s, unicode):
s = s.encode(encoding)
result = ''
for c in s:
if not (32 <= ord(c) < 127) or c in ('\\', '"'):
result += '\\%03o' % ord(c)
else:
result += c
return '"' + result + '"'
types = (make_type('Counter', []), make_type('Histogram', ['max', 'buckets']),)
inst_map = dict((t[0].__name__, t[1]) for t in types)
stats = []
for attr in attrs:
found = False
for t, lst in types:
t_name = t.__name__.lower()
if t_name in attr:
name = attr[t_name]
del attr[t_name]
lst.append(t(name=name, **attr))
found = True
break
assert found, "Bad decl: %s" % attr
found = False
for t, lst in types:
t_name = t.__name__.lower()
if t_name in attr:
name = attr[t_name]
del attr[t_name]
lst.append(t(name=name, **attr))
found = True
break
assert found, "Bad decl: %s" % attr
def dbl2u64(d):
return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value
return ctypes.c_ulonglong.from_buffer(ctypes.c_double(d)).value
def shift_works_until(mapped_bounds, shift_bits):
for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])):
a, b = ab
if (a >> shift_bits) == (b >> shift_bits):
return i
return len(mapped_bounds)
for i, ab in enumerate(zip(mapped_bounds, mapped_bounds[1:])):
a, b = ab
if (a >> shift_bits) == (b >> shift_bits):
return i
return len(mapped_bounds)
def find_ideal_shift(mapped_bounds, max_size):
best = None
for shift_bits in reversed(range(0,64)):
n = shift_works_until(mapped_bounds, shift_bits)
if n == 0: continue
table_size = mapped_bounds[n-1] >> shift_bits
if table_size > max_size: continue
if table_size > 65535: continue
if best is None:
best = (shift_bits, n, table_size)
elif best[1] < n:
best = (shift_bits, n, table_size)
print best
return best
best = None
for shift_bits in reversed(range(0, 64)):
n = shift_works_until(mapped_bounds, shift_bits)
if n == 0: continue
table_size = mapped_bounds[n - 1] >> shift_bits
if table_size > max_size: continue
if table_size > 65535: continue
if best is None:
best = (shift_bits, n, table_size)
elif best[1] < n:
best = (shift_bits, n, table_size)
print best
return best
def gen_map_table(mapped_bounds, shift_data):
tbl = []
cur = 0
print mapped_bounds
mapped_bounds = [x >> shift_data[0] for x in mapped_bounds]
print mapped_bounds
for i in range(0, mapped_bounds[shift_data[1]-1]):
while i > mapped_bounds[cur]:
cur += 1
tbl.append(cur)
return tbl
tbl = []
cur = 0
print mapped_bounds
mapped_bounds = [x >> shift_data[0] for x in mapped_bounds]
print mapped_bounds
for i in range(0, mapped_bounds[shift_data[1] - 1]):
while i > mapped_bounds[cur]:
cur += 1
tbl.append(cur)
return tbl
static_tables = []
def decl_static_table(values, type):
global static_tables
v = (type, values)
for i, vp in enumerate(static_tables):
if v == vp: return i
print "ADD TABLE: %s %r" % (type, values)
r = len(static_tables)
static_tables.append(v)
return r
global static_tables
v = (type, values)
for i, vp in enumerate(static_tables):
if v == vp: return i
print "ADD TABLE: %s %r" % (type, values)
r = len(static_tables)
static_tables.append(v)
return r
def type_for_uint_table(table):
mv = max(table)
if mv < 2**8:
return 'uint8_t'
elif mv < 2**16:
return 'uint16_t'
elif mv < 2**32:
return 'uint32_t'
else:
return 'uint64_t'
mv = max(table)
if mv < 2**8:
return 'uint8_t'
elif mv < 2**16:
return 'uint16_t'
elif mv < 2**32:
return 'uint32_t'
else:
return 'uint64_t'
def gen_bucket_code(histogram):
bounds = [0, 1]
done_trivial = False
done_unmapped = False
first_nontrivial = None
first_unmapped = None
while len(bounds) < histogram.buckets + 1:
if len(bounds) == histogram.buckets:
nextb = int(histogram.max)
bounds = [0, 1]
done_trivial = False
done_unmapped = False
first_nontrivial = None
first_unmapped = None
while len(bounds) < histogram.buckets + 1:
if len(bounds) == histogram.buckets:
nextb = int(histogram.max)
else:
mul = math.pow(
float(histogram.max) / bounds[-1],
1.0 / (histogram.buckets + 1 - len(bounds)))
nextb = int(math.ceil(bounds[-1] * mul))
if nextb <= bounds[-1] + 1:
nextb = bounds[-1] + 1
elif not done_trivial:
done_trivial = True
first_nontrivial = len(bounds)
bounds.append(nextb)
bounds_idx = decl_static_table(bounds, 'int')
if done_trivial:
first_nontrivial_code = dbl2u64(first_nontrivial)
code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds]
shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 *
histogram.buckets)
#print first_nontrivial, shift_data, bounds
#if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
if first_nontrivial is None:
code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' %
histogram.name.upper())
else:
mul = math.pow(float(histogram.max) / bounds[-1],
1.0 / (histogram.buckets + 1 - len(bounds)))
nextb = int(math.ceil(bounds[-1] * mul))
if nextb <= bounds[-1] + 1:
nextb = bounds[-1] + 1
elif not done_trivial:
done_trivial = True
first_nontrivial = len(bounds)
bounds.append(nextb)
bounds_idx = decl_static_table(bounds, 'int')
if done_trivial:
first_nontrivial_code = dbl2u64(first_nontrivial)
code_bounds = [dbl2u64(x) - first_nontrivial_code for x in bounds]
shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets)
#print first_nontrivial, shift_data, bounds
#if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
if first_nontrivial is None:
code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n'
% histogram.name.upper())
else:
code += 'if (value < %d) {\n' % first_nontrivial
code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n'
% histogram.name.upper())
code += 'return;\n'
code += '}'
first_nontrivial_code = dbl2u64(first_nontrivial)
if shift_data is not None:
map_table_idx = decl_static_table(map_table, type_for_uint_table(map_table))
code += 'union { double dbl; uint64_t uint; } _val, _bkt;\n'
code += '_val.dbl = value;\n'
code += 'if (_val.uint < %dull) {\n' % ((map_table[-1] << shift_data[0]) + first_nontrivial_code)
code += 'int bucket = '
code += 'grpc_stats_table_%d[((_val.uint - %dull) >> %d)] + %d;\n' % (map_table_idx, first_nontrivial_code, shift_data[0], first_nontrivial)
code += '_bkt.dbl = grpc_stats_table_%d[bucket];\n' % bounds_idx
code += 'bucket -= (_val.uint < _bkt.uint);\n'
code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, bucket);\n' % histogram.name.upper()
code += 'return;\n'
code += '}\n'
code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, '% histogram.name.upper()
code += 'grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_%d, %d));\n' % (bounds_idx, histogram.buckets)
return (code, bounds_idx)
code += 'if (value < %d) {\n' % first_nontrivial
code += ('GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, value);\n' %
histogram.name.upper())
code += 'return;\n'
code += '}'
first_nontrivial_code = dbl2u64(first_nontrivial)
if shift_data is not None:
map_table_idx = decl_static_table(map_table,
type_for_uint_table(map_table))
code += 'union { double dbl; uint64_t uint; } _val, _bkt;\n'
code += '_val.dbl = value;\n'
code += 'if (_val.uint < %dull) {\n' % (
(map_table[-1] << shift_data[0]) + first_nontrivial_code)
code += 'int bucket = '
code += 'grpc_stats_table_%d[((_val.uint - %dull) >> %d)] + %d;\n' % (
map_table_idx, first_nontrivial_code, shift_data[0],
first_nontrivial)
code += '_bkt.dbl = grpc_stats_table_%d[bucket];\n' % bounds_idx
code += 'bucket -= (_val.uint < _bkt.uint);\n'
code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, bucket);\n' % histogram.name.upper(
)
code += 'return;\n'
code += '}\n'
code += 'GRPC_STATS_INC_HISTOGRAM(GRPC_STATS_HISTOGRAM_%s, ' % histogram.name.upper(
)
code += 'grpc_stats_histo_find_bucket_slow(value, grpc_stats_table_%d, %d));\n' % (
bounds_idx, histogram.buckets)
return (code, bounds_idx)
# utility: print a big comment block into a set of files
def put_banner(files, banner):
for f in files:
print >>f, '/*'
for line in banner:
print >>f, ' * %s' % line
print >>f, ' */'
print >>f
for f in files:
print >> f, '/*'
for line in banner:
print >> f, ' * %s' % line
print >> f, ' */'
print >> f
with open('src/core/lib/debug/stats_data.h', 'w') as H:
# copy-paste copyright notice from this file
with open(sys.argv[0]) as my_source:
copyright = []
for line in my_source:
if line[0] != '#': break
for line in my_source:
if line[0] == '#':
copyright.append(line)
break
for line in my_source:
if line[0] != '#':
break
copyright.append(line)
put_banner([H], [line[2:].rstrip() for line in copyright])
put_banner([H], ["Automatically generated by tools/codegen/core/gen_stats_data.py"])
print >>H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
print >>H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
print >>H
print >>H, "#include <inttypes.h>"
print >>H, "#include \"src/core/lib/iomgr/exec_ctx.h\""
print >>H
print >>H, "#ifdef __cplusplus"
print >>H, "extern \"C\" {"
print >>H, "#endif"
print >>H
for typename, instances in sorted(inst_map.items()):
print >>H, "typedef enum {"
for inst in instances:
print >>H, " GRPC_STATS_%s_%s," % (typename.upper(), inst.name.upper())
print >>H, " GRPC_STATS_%s_COUNT" % (typename.upper())
print >>H, "} grpc_stats_%ss;" % (typename.lower())
print >>H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % (
typename.lower(), typename.upper())
print >>H, "extern const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT];" % (
typename.lower(), typename.upper())
histo_start = []
histo_buckets = []
histo_bucket_boundaries = []
print >>H, "typedef enum {"
first_slot = 0
for histogram in inst_map['Histogram']:
histo_start.append(first_slot)
histo_buckets.append(histogram.buckets)
print >>H, " GRPC_STATS_HISTOGRAM_%s_FIRST_SLOT = %d," % (histogram.name.upper(), first_slot)
print >>H, " GRPC_STATS_HISTOGRAM_%s_BUCKETS = %d," % (histogram.name.upper(), histogram.buckets)
first_slot += histogram.buckets
print >>H, " GRPC_STATS_HISTOGRAM_BUCKETS = %d" % first_slot
print >>H, "} grpc_stats_histogram_constants;"
for ctr in inst_map['Counter']:
print >>H, ("#define GRPC_STATS_INC_%s() " +
"GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_%s)") % (
ctr.name.upper(), ctr.name.upper())
for histogram in inst_map['Histogram']:
print >>H, "#define GRPC_STATS_INC_%s(value) grpc_stats_inc_%s( (int)(value))" % (
histogram.name.upper(), histogram.name.lower())
print >>H, "void grpc_stats_inc_%s(int x);" % histogram.name.lower()
for i, tbl in enumerate(static_tables):
print >>H, "extern const %s grpc_stats_table_%d[%d];" % (tbl[0], i, len(tbl[1]))
print >>H, "extern const int grpc_stats_histo_buckets[%d];" % len(inst_map['Histogram'])
print >>H, "extern const int grpc_stats_histo_start[%d];" % len(inst_map['Histogram'])
print >>H, "extern const int *const grpc_stats_histo_bucket_boundaries[%d];" % len(inst_map['Histogram'])
print >>H, "extern void (*const grpc_stats_inc_histogram[%d])(int x);" % len(inst_map['Histogram'])
print >>H
print >>H, "#ifdef __cplusplus"
print >>H, "}"
print >>H, "#endif"
print >>H
print >>H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */"
# copy-paste copyright notice from this file
with open(sys.argv[0]) as my_source:
copyright = []
for line in my_source:
if line[0] != '#': break
for line in my_source:
if line[0] == '#':
copyright.append(line)
break
for line in my_source:
if line[0] != '#':
break
copyright.append(line)
put_banner([H], [line[2:].rstrip() for line in copyright])
put_banner(
[H],
["Automatically generated by tools/codegen/core/gen_stats_data.py"])
print >> H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
print >> H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
print >> H
print >> H, "#include <inttypes.h>"
print >> H, "#include \"src/core/lib/iomgr/exec_ctx.h\""
print >> H
print >> H, "#ifdef __cplusplus"
print >> H, "extern \"C\" {"
print >> H, "#endif"
print >> H
for typename, instances in sorted(inst_map.items()):
print >> H, "typedef enum {"
for inst in instances:
print >> H, " GRPC_STATS_%s_%s," % (typename.upper(),
inst.name.upper())
print >> H, " GRPC_STATS_%s_COUNT" % (typename.upper())
print >> H, "} grpc_stats_%ss;" % (typename.lower())
print >> H, "extern const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT];" % (
typename.lower(), typename.upper())
print >> H, "extern const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT];" % (
typename.lower(), typename.upper())
histo_start = []
histo_buckets = []
histo_bucket_boundaries = []
print >> H, "typedef enum {"
first_slot = 0
for histogram in inst_map['Histogram']:
histo_start.append(first_slot)
histo_buckets.append(histogram.buckets)
print >> H, " GRPC_STATS_HISTOGRAM_%s_FIRST_SLOT = %d," % (
histogram.name.upper(), first_slot)
print >> H, " GRPC_STATS_HISTOGRAM_%s_BUCKETS = %d," % (
histogram.name.upper(), histogram.buckets)
first_slot += histogram.buckets
print >> H, " GRPC_STATS_HISTOGRAM_BUCKETS = %d" % first_slot
print >> H, "} grpc_stats_histogram_constants;"
for ctr in inst_map['Counter']:
print >> H, ("#define GRPC_STATS_INC_%s() " +
"GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_%s)") % (
ctr.name.upper(), ctr.name.upper())
for histogram in inst_map['Histogram']:
print >> H, "#define GRPC_STATS_INC_%s(value) grpc_stats_inc_%s( (int)(value))" % (
histogram.name.upper(), histogram.name.lower())
print >> H, "void grpc_stats_inc_%s(int x);" % histogram.name.lower()
for i, tbl in enumerate(static_tables):
print >> H, "extern const %s grpc_stats_table_%d[%d];" % (tbl[0], i,
len(tbl[1]))
print >> H, "extern const int grpc_stats_histo_buckets[%d];" % len(
inst_map['Histogram'])
print >> H, "extern const int grpc_stats_histo_start[%d];" % len(
inst_map['Histogram'])
print >> H, "extern const int *const grpc_stats_histo_bucket_boundaries[%d];" % len(
inst_map['Histogram'])
print >> H, "extern void (*const grpc_stats_inc_histogram[%d])(int x);" % len(
inst_map['Histogram'])
print >> H
print >> H, "#ifdef __cplusplus"
print >> H, "}"
print >> H, "#endif"
print >> H
print >> H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */"
with open('src/core/lib/debug/stats_data.cc', 'w') as C:
# copy-paste copyright notice from this file
with open(sys.argv[0]) as my_source:
copyright = []
for line in my_source:
if line[0] != '#': break
for line in my_source:
if line[0] == '#':
copyright.append(line)
break
for line in my_source:
if line[0] != '#':
break
copyright.append(line)
put_banner([C], [line[2:].rstrip() for line in copyright])
put_banner([C], ["Automatically generated by tools/codegen/core/gen_stats_data.py"])
print >>C, "#include \"src/core/lib/debug/stats_data.h\""
print >>C, "#include \"src/core/lib/debug/stats.h\""
print >>C, "#include \"src/core/lib/iomgr/exec_ctx.h\""
print >>C, "#include <grpc/support/useful.h>"
histo_code = []
for histogram in inst_map['Histogram']:
code, bounds_idx = gen_bucket_code(histogram)
histo_bucket_boundaries.append(bounds_idx)
histo_code.append(code)
for typename, instances in sorted(inst_map.items()):
print >>C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % (
typename.lower(), typename.upper())
for inst in instances:
print >>C, " %s," % c_str(inst.name)
print >>C, "};"
print >>C, "const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT] = {" % (
typename.lower(), typename.upper())
for inst in instances:
print >>C, " %s," % c_str(inst.doc)
print >>C, "};"
for i, tbl in enumerate(static_tables):
print >>C, "const %s grpc_stats_table_%d[%d] = {%s};" % (
tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1]))
for histogram, code in zip(inst_map['Histogram'], histo_code):
print >>C, ("void grpc_stats_inc_%s(int value) {%s}") % (
histogram.name.lower(),
code)
print >>C, "const int grpc_stats_histo_buckets[%d] = {%s};" % (
len(inst_map['Histogram']), ','.join('%s' % x for x in histo_buckets))
print >>C, "const int grpc_stats_histo_start[%d] = {%s};" % (
len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start))
print >>C, "const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % (
len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries))
print >>C, "void (*const grpc_stats_inc_histogram[%d])(int x) = {%s};" % (
len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram']))
# copy-paste copyright notice from this file
with open(sys.argv[0]) as my_source:
copyright = []
for line in my_source:
if line[0] != '#': break
for line in my_source:
if line[0] == '#':
copyright.append(line)
break
for line in my_source:
if line[0] != '#':
break
copyright.append(line)
put_banner([C], [line[2:].rstrip() for line in copyright])
put_banner(
[C],
["Automatically generated by tools/codegen/core/gen_stats_data.py"])
print >> C, "#include \"src/core/lib/debug/stats_data.h\""
print >> C, "#include \"src/core/lib/debug/stats.h\""
print >> C, "#include \"src/core/lib/iomgr/exec_ctx.h\""
print >> C, "#include <grpc/support/useful.h>"
histo_code = []
for histogram in inst_map['Histogram']:
code, bounds_idx = gen_bucket_code(histogram)
histo_bucket_boundaries.append(bounds_idx)
histo_code.append(code)
for typename, instances in sorted(inst_map.items()):
print >> C, "const char *grpc_stats_%s_name[GRPC_STATS_%s_COUNT] = {" % (
typename.lower(), typename.upper())
for inst in instances:
print >> C, " %s," % c_str(inst.name)
print >> C, "};"
print >> C, "const char *grpc_stats_%s_doc[GRPC_STATS_%s_COUNT] = {" % (
typename.lower(), typename.upper())
for inst in instances:
print >> C, " %s," % c_str(inst.doc)
print >> C, "};"
for i, tbl in enumerate(static_tables):
print >> C, "const %s grpc_stats_table_%d[%d] = {%s};" % (
tbl[0], i, len(tbl[1]), ','.join('%s' % x for x in tbl[1]))
for histogram, code in zip(inst_map['Histogram'], histo_code):
print >> C, ("void grpc_stats_inc_%s(int value) {%s}") % (
histogram.name.lower(), code)
print >> C, "const int grpc_stats_histo_buckets[%d] = {%s};" % (
len(inst_map['Histogram']), ','.join('%s' % x for x in histo_buckets))
print >> C, "const int grpc_stats_histo_start[%d] = {%s};" % (
len(inst_map['Histogram']), ','.join('%s' % x for x in histo_start))
print >> C, "const int *const grpc_stats_histo_bucket_boundaries[%d] = {%s};" % (
len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x
for x in histo_bucket_boundaries))
print >> C, "void (*const grpc_stats_inc_histogram[%d])(int x) = {%s};" % (
len(inst_map['Histogram']),
','.join('grpc_stats_inc_%s' % histogram.name.lower()
for histogram in inst_map['Histogram']))
# patch qps_test bigquery schema
RECORD_EXPLICIT_PERCENTILES = [50, 95, 99]
with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f:
qps_schema = json.loads(f.read())
qps_schema = json.loads(f.read())
def FindNamed(js, name):
for el in js:
if el['name'] == name:
return el
for el in js:
if el['name'] == name:
return el
def RemoveCoreFields(js):
new_fields = []
for field in js['fields']:
if not field['name'].startswith('core_'):
new_fields.append(field)
js['fields'] = new_fields
new_fields = []
for field in js['fields']:
if not field['name'].startswith('core_'):
new_fields.append(field)
js['fields'] = new_fields
RemoveCoreFields(FindNamed(qps_schema, 'clientStats'))
RemoveCoreFields(FindNamed(qps_schema, 'serverStats'))
def AddCoreFields(js):
for counter in inst_map['Counter']:
js['fields'].append({
'name': 'core_%s' % counter.name,
'type': 'INTEGER',
'mode': 'NULLABLE'
})
for histogram in inst_map['Histogram']:
js['fields'].append({
'name': 'core_%s' % histogram.name,
'type': 'STRING',
'mode': 'NULLABLE'
})
js['fields'].append({
'name': 'core_%s_bkts' % histogram.name,
'type': 'STRING',
'mode': 'NULLABLE'
})
for pctl in RECORD_EXPLICIT_PERCENTILES:
js['fields'].append({
'name': 'core_%s_%dp' % (histogram.name, pctl),
'type': 'FLOAT',
'mode': 'NULLABLE'
})
for counter in inst_map['Counter']:
js['fields'].append({
'name': 'core_%s' % counter.name,
'type': 'INTEGER',
'mode': 'NULLABLE'
})
for histogram in inst_map['Histogram']:
js['fields'].append({
'name': 'core_%s' % histogram.name,
'type': 'STRING',
'mode': 'NULLABLE'
})
js['fields'].append({
'name': 'core_%s_bkts' % histogram.name,
'type': 'STRING',
'mode': 'NULLABLE'
})
for pctl in RECORD_EXPLICIT_PERCENTILES:
js['fields'].append({
'name': 'core_%s_%dp' % (histogram.name, pctl),
'type': 'FLOAT',
'mode': 'NULLABLE'
})
AddCoreFields(FindNamed(qps_schema, 'clientStats'))
AddCoreFields(FindNamed(qps_schema, 'serverStats'))
with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f:
f.write(json.dumps(qps_schema, indent=2, sort_keys=True))
f.write(json.dumps(qps_schema, indent=2, sort_keys=True))
# and generate a helper script to massage scenario results into the format we'd
# like to query
with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P:
with open(sys.argv[0]) as my_source:
for line in my_source:
if line[0] != '#': break
for line in my_source:
if line[0] == '#':
print >>P, line.rstrip()
break
for line in my_source:
if line[0] != '#':
break
print >>P, line.rstrip()
print >>P
print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py'
print >>P
print >>P, 'import massage_qps_stats_helpers'
print >>P, 'def massage_qps_stats(scenario_result):'
print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:'
print >>P, ' if "coreStats" not in stats: return'
print >>P, ' core_stats = stats["coreStats"]'
print >>P, ' del stats["coreStats"]'
for counter in inst_map['Counter']:
print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name)
for i, histogram in enumerate(inst_map['Histogram']):
print >>P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name
print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name
print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name
for pctl in RECORD_EXPLICIT_PERCENTILES:
print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % (
histogram.name, pctl, pctl)
with open(sys.argv[0]) as my_source:
for line in my_source:
if line[0] != '#': break
for line in my_source:
if line[0] == '#':
print >> P, line.rstrip()
break
for line in my_source:
if line[0] != '#':
break
print >> P, line.rstrip()
print >> P
print >> P, '# Autogenerated by tools/codegen/core/gen_stats_data.py'
print >> P
print >> P, 'import massage_qps_stats_helpers'
print >> P, 'def massage_qps_stats(scenario_result):'
print >> P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:'
print >> P, ' if "coreStats" not in stats: return'
print >> P, ' core_stats = stats["coreStats"]'
print >> P, ' del stats["coreStats"]'
for counter in inst_map['Counter']:
print >> P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (
counter.name, counter.name)
for i, histogram in enumerate(inst_map['Histogram']):
print >> P, ' h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name
print >> P, ' stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name
print >> P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name
for pctl in RECORD_EXPLICIT_PERCENTILES:
print >> P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % (
histogram.name, pctl, pctl)
with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S:
columns = []
for counter in inst_map['Counter']:
columns.append(('%s_per_iteration' % counter.name, 'FLOAT'))
print >>S, ',\n'.join('%s:%s' % x for x in columns)
columns = []
for counter in inst_map['Counter']:
columns.append(('%s_per_iteration' % counter.name, 'FLOAT'))
print >> S, ',\n'.join('%s:%s' % x for x in columns)

1
tools/distrib/yapf_code.sh
Unescape View File

@ -21,6 +21,7 @@ cd "$(dirname "${0}")/../.."
DIRS=(
'src/python'
'tools/buildgen'
'tools/codegen'
)
EXCLUSIONS=(
'grpcio/grpc_*.py'

Write Preview
Loading…
Cancel
Save