Upload core stats to qps_test

pull/12509/head
Craig Tiller 7 years ago
parent a06e5fdfb7
commit 73a6170896
  1. 87
      tools/codegen/core/gen_stats_data.py
  2. 2
      tools/run_tests/performance/bq_upload_result.py
  3. 105
      tools/run_tests/performance/massage_qps_stats.py
  4. 57
      tools/run_tests/performance/massage_qps_stats_helpers.py
  5. 1296
      tools/run_tests/performance/scenario_result_schema.json

@ -313,3 +313,90 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C:
len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries))
print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % (
len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram']))
# patch qps_test bigquery schema
RECORD_EXPLICIT_PERCENTILES = [50, 95, 99]
with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f:
qps_schema = json.loads(f.read())
def FindNamed(js, name):
for el in js:
if el['name'] == name:
return el
def RemoveCoreFields(js):
new_fields = []
for field in js['fields']:
if not field['name'].startswith('core_'):
new_fields.append(field)
js['fields'] = new_fields
RemoveCoreFields(FindNamed(qps_schema, 'clientStats'))
RemoveCoreFields(FindNamed(qps_schema, 'serverStats'))
def AddCoreFields(js):
for counter in inst_map['Counter']:
js['fields'].append({
'name': 'core_%s' % counter.name,
'type': 'INTEGER',
'mode': 'NULLABLE'
})
for histogram in inst_map['Histogram']:
js['fields'].append({
'name': 'core_%s' % histogram.name,
'type': 'STRING',
'mode': 'NULLABLE'
})
js['fields'].append({
'name': 'core_%s_bkts' % histogram.name,
'type': 'STRING',
'mode': 'NULLABLE'
})
for pctl in RECORD_EXPLICIT_PERCENTILES:
js['fields'].append({
'name': 'core_%s_%dp' % (histogram.name, pctl),
'type': 'FLOAT',
'mode': 'NULLABLE'
})
AddCoreFields(FindNamed(qps_schema, 'clientStats'))
AddCoreFields(FindNamed(qps_schema, 'serverStats'))
with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f:
f.write(json.dumps(qps_schema, indent=2, sort_keys=True))
# and generate a helper script to massage scenario results into the format we'd
# like to query
with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P:
with open(sys.argv[0]) as my_source:
for line in my_source:
if line[0] != '#': break
for line in my_source:
if line[0] == '#':
print >>P, line.rstrip()
break
for line in my_source:
if line[0] != '#':
break
print >>P, line.rstrip()
print >>P
print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py'
print >>P
print >>P, 'import massage_qps_stats_helpers'
print >>P, 'def massage_qps_stats(scenario_result):'
print >>P, ' for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:'
print >>P, ' if "coreStats" not in stats: return'
print >>P, ' core_stats = stats["coreStats"]'
print >>P, ' del stats["coreStats"]'
for counter in inst_map['Counter']:
print >>P, ' stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name)
for i, histogram in enumerate(inst_map['Histogram']):
print >>P, ' stats["core_%s"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").buckets)' % (histogram.name, histogram.name)
print >>P, ' stats["core_%s_bkts"] = ",".join("%%f" %% x for x in massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (histogram.name, histogram.name)
for pctl in RECORD_EXPLICIT_PERCENTILES:
print >>P, ' stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "%s").buckets, %d, massage_qps_stats_helpers.histogram(core_stats, "%s").boundaries)' % (
histogram.name, pctl, histogram.name, pctl, histogram.name)

@ -24,6 +24,7 @@ import os
import sys
import time
import uuid
import massage_qps_stats
gcp_utils_dir = os.path.abspath(os.path.join(
@ -117,6 +118,7 @@ def _flatten_result_inplace(scenario_result):
scenario_result['serverCpuUsage'] = scenario_result['summary'].pop('serverCpuUsage', None)
scenario_result['summary'].pop('successfulRequestsPerSecond', None)
scenario_result['summary'].pop('failedRequestsPerSecond', None)
massage_qps_stats.massage_qps_stats(scenario_result)
def _populate_metadata_inplace(scenario_result):

@ -0,0 +1,105 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Autogenerated by tools/codegen/core/gen_stats_data.py
import massage_qps_stats_helpers
def massage_qps_stats(scenario_result):
for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:
if "coreStats" not in stats: return
core_stats = stats["coreStats"]
del stats["coreStats"]
stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created")
stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created")
stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll")
stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait")
stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups")
stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write")
stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read")
stats["core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_pollers_created")
stats["core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_poller_polls")
stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_batches")
stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_cancel")
stats["core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_initial_metadata")
stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_message")
stats["core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_trailing_metadata")
stats["core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_initial_metadata")
stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_message")
stats["core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_trailing_metadata")
stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_settings_writes")
stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_pings_sent")
stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_begun")
stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded")
stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued")
stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes")
stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated")
stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items")
stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items")
stats["core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_offloaded")
stats["core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_short_items")
stats["core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_long_items")
stats["core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_to_self")
stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated")
stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained")
stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries")
stats["core_tcp_write_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets)
stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries)
stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries)
stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries)
stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size").boundaries)
stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets)
stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries)
stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries)
stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries)
stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size").boundaries)
stats["core_tcp_read_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets)
stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries)
stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries)
stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries)
stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size").boundaries)
stats["core_tcp_read_offer"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets)
stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries)
stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries)
stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries)
stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer").boundaries)
stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets)
stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries)
stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries)
stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries)
stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size").boundaries)
stats["core_http2_send_message_size"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets)
stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries)
stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries)
stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries)
stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size").boundaries)
stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets)
stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries)
stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries)
stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries)
stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write").boundaries)
stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets)
stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries)
stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries)
stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries)
stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write").boundaries)
stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets)
stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries)
stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries)
stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries)
stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write").boundaries)
stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets)
stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries)
stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 50, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries)
stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 95, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries)
stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").buckets, 99, massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write").boundaries)

@ -0,0 +1,57 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
def threshold_for_count_below(buckets, boundaries, count_below):
count_so_far = 0
for lower_idx in range(0, len(buckets)):
count_so_far += buckets[lower_idx]
if count_so_far >= count_below:
break
if count_so_far == count_below:
# this bucket hits the threshold exactly... we should be midway through
# any run of zero values following the bucket
for upper_idx in range(lower_idx + 1, num_buckets):
if buckets[upper_idx] != 0:
break
return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0
else:
# treat values as uniform throughout the bucket, and find where this value
# should lie
lower_bound = boundaries[lower_idx]
upper_bound = boundaries[lower_idx + 1]
return (upper_bound -
(upper_bound - lower_bound) * (count_so_far - count_below) /
float(buckets[lower_idx]))
def percentile(buckets, pctl, boundaries):
return threshold_for_count_below(
buckets, boundaries, sum(buckets) * pctl / 100.0)
def counter(core_stats, name):
for stat in core_stats['metrics']:
if stat['name'] == name:
return int(stat.get('count', 0))
Histogram = collections.namedtuple('Histogram', 'buckets boundaries')
def histogram(core_stats, name):
for stat in core_stats['metrics']:
if stat['name'] == name:
buckets = []
boundaries = []
for b in stat['histogram']['buckets']:
buckets.append(int(b.get('count', 0)))
boundaries.append(int(b.get('start', 0)))
return Histogram(buckets=buckets, boundaries=boundaries)

File diff suppressed because it is too large Load Diff
Loading…
Cancel
Save