|
|
|
#!/usr/bin/env python2.7
|
|
|
|
#
|
|
|
|
# Copyright 2017 gRPC authors.
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
""" Computes the diff between two qps runs and outputs significant results """
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
import json
|
|
|
|
import multiprocessing
|
|
|
|
import os
|
|
|
|
import qps_scenarios
|
|
|
|
import shutil
|
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
import tabulate
|
|
|
|
|
|
|
|
sys.path.append(
|
|
|
|
os.path.join(
|
|
|
|
os.path.dirname(sys.argv[0]), '..', 'microbenchmarks', 'bm_diff'))
|
|
|
|
import bm_speedup
|
|
|
|
|
|
|
|
sys.path.append(
|
|
|
|
os.path.join(
|
|
|
|
os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
|
|
|
|
import comment_on_pr
|
|
|
|
|
|
|
|
|
|
|
|
def _args():
|
|
|
|
argp = argparse.ArgumentParser(description='Perform diff on QPS Driver')
|
|
|
|
argp.add_argument(
|
|
|
|
'-d',
|
|
|
|
'--diff_base',
|
|
|
|
type=str,
|
|
|
|
help='Commit or branch to compare the current one to')
|
|
|
|
argp.add_argument(
|
|
|
|
'-l',
|
|
|
|
'--loops',
|
|
|
|
type=int,
|
|
|
|
default=4,
|
|
|
|
help='Number of loops for each benchmark. More loops cuts down on noise'
|
|
|
|
)
|
|
|
|
argp.add_argument(
|
|
|
|
'-j',
|
|
|
|
'--jobs',
|
|
|
|
type=int,
|
|
|
|
default=multiprocessing.cpu_count(),
|
|
|
|
help='Number of CPUs to use')
|
|
|
|
args = argp.parse_args()
|
|
|
|
assert args.diff_base, "diff_base must be set"
|
|
|
|
return args
|
|
|
|
|
|
|
|
|
|
|
|
def _make_cmd(jobs):
|
|
|
|
return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker']
|
|
|
|
|
|
|
|
|
|
|
|
def build(name, jobs):
|
|
|
|
shutil.rmtree('qps_diff_%s' % name, ignore_errors=True)
|
|
|
|
subprocess.check_call(['git', 'submodule', 'update'])
|
|
|
|
try:
|
|
|
|
subprocess.check_call(_make_cmd(jobs))
|
|
|
|
except subprocess.CalledProcessError, e:
|
|
|
|
subprocess.check_call(['make', 'clean'])
|
|
|
|
subprocess.check_call(_make_cmd(jobs))
|
|
|
|
os.rename('bins', 'qps_diff_%s' % name)
|
|
|
|
|
|
|
|
|
|
|
|
def _run_cmd(name, scenario, fname):
|
|
|
|
return [
|
|
|
|
'qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario,
|
|
|
|
'--json_file_out', fname
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
def run(name, scenarios, loops):
|
|
|
|
for sn in scenarios:
|
|
|
|
for i in range(0, loops):
|
|
|
|
fname = "%s.%s.%d.json" % (sn, name, i)
|
|
|
|
subprocess.check_call(_run_cmd(name, scenarios[sn], fname))
|
|
|
|
|
|
|
|
|
|
|
|
def _load_qps(fname):
|
|
|
|
try:
|
|
|
|
with open(fname) as f:
|
|
|
|
return json.loads(f.read())['qps']
|
|
|
|
except IOError, e:
|
|
|
|
print("IOError occurred reading file: %s" % fname)
|
|
|
|
return None
|
|
|
|
except ValueError, e:
|
|
|
|
print("ValueError occurred reading file: %s" % fname)
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
def _median(ary):
|
|
|
|
assert (len(ary))
|
|
|
|
ary = sorted(ary)
|
|
|
|
n = len(ary)
|
|
|
|
if n % 2 == 0:
|
|
|
|
return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
|
|
|
|
else:
|
|
|
|
return ary[n / 2]
|
|
|
|
|
|
|
|
|
|
|
|
def diff(scenarios, loops, old, new):
|
|
|
|
old_data = {}
|
|
|
|
new_data = {}
|
|
|
|
|
|
|
|
# collect data
|
|
|
|
for sn in scenarios:
|
|
|
|
old_data[sn] = []
|
|
|
|
new_data[sn] = []
|
|
|
|
for i in range(loops):
|
|
|
|
old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i)))
|
|
|
|
new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i)))
|
|
|
|
|
|
|
|
# crunch data
|
|
|
|
headers = ['Benchmark', 'qps']
|
|
|
|
rows = []
|
|
|
|
for sn in scenarios:
|
|
|
|
mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn]))
|
|
|
|
print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old,
|
|
|
|
old_data[sn], mdn_diff))
|
|
|
|
s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5)
|
|
|
|
if abs(s) > 3 and mdn_diff > 0.5:
|
|
|
|
rows.append([sn, '%+d%%' % s])
|
|
|
|
|
|
|
|
if rows:
|
|
|
|
return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
|
|
|
|
else:
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
def main(args):
|
|
|
|
build('new', args.jobs)
|
|
|
|
|
|
|
|
if args.diff_base:
|
|
|
|
where_am_i = subprocess.check_output(
|
|
|
|
['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
|
|
|
|
subprocess.check_call(['git', 'checkout', args.diff_base])
|
|
|
|
try:
|
|
|
|
build('old', args.jobs)
|
|
|
|
finally:
|
|
|
|
subprocess.check_call(['git', 'checkout', where_am_i])
|
|
|
|
subprocess.check_call(['git', 'submodule', 'update'])
|
|
|
|
|
|
|
|
run('new', qps_scenarios._SCENARIOS, args.loops)
|
|
|
|
run('old', qps_scenarios._SCENARIOS, args.loops)
|
|
|
|
|
|
|
|
diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new')
|
|
|
|
|
|
|
|
if diff_output:
|
|
|
|
text = '[qps] Performance differences noted:\n%s' % diff_output
|
|
|
|
else:
|
|
|
|
text = '[qps] No significant performance differences'
|
|
|
|
print('%s' % text)
|
|
|
|
comment_on_pr.comment_on_pr('```\n%s\n```' % text)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
args = _args()
|
|
|
|
main(args)
|