diff --git a/tools/profiling/microbenchmarks/README.md b/tools/profiling/microbenchmarks/README.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/profiling/microbenchmarks/bm_build.py b/tools/profiling/microbenchmarks/bm_build.py index 13ca1dfd1b4..a5d1ec34475 100755 --- a/tools/profiling/microbenchmarks/bm_build.py +++ b/tools/profiling/microbenchmarks/bm_build.py @@ -45,24 +45,24 @@ def _args(): argp.add_argument('-n', '--name', type=str, help='Unique name of this build') return argp.parse_args() -def _make_cmd(cfg, jobs, benchmarks): +def _make_cmd(cfg, benchmarks, jobs): return ['make'] + benchmarks + [ 'CONFIG=%s' % cfg, '-j', '%d' % jobs] -def build(name, jobs, benchmarks): +def build(name, benchmarks, jobs): shutil.rmtree('bm_diff_%s' % name, ignore_errors=True) subprocess.check_call(['git', 'submodule', 'update']) try: - subprocess.check_call(_make_cmd('opt', jobs, benchmarks)) - subprocess.check_call(_make_cmd('counters', jobs, benchmarks)) + subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) except subprocess.CalledProcessError, e: subprocess.check_call(['make', 'clean']) - subprocess.check_call(_make_cmd('opt', jobs, benchmarks)) - subprocess.check_call(_make_cmd('counters', jobs, benchmarks)) + subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) os.rename('bins', 'bm_diff_%s' % name, ) if __name__ == '__main__': args = _args() - build(args.name, args.jobs, args.benchmarks) + build(args.name, args.benchmarks, args.jobs) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 4de318fec7c..f68dff840ec 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -41,7 +41,7 @@ import collections verbose = False -def median(ary): +def _median(ary): ary = sorted(ary) n = len(ary) if n%2 == 0: @@ -68,7 +68,7 @@ def _args(): assert args.old return args -def maybe_print(str): +def _maybe_print(str): if verbose: print str class Benchmark: @@ -85,14 +85,15 @@ class Benchmark: if f in data: self.samples[new][f].append(float(data[f])) - def process(self, track): + def process(self, track, new_name, old_name): for f in sorted(track): new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue - mdn_diff = abs(median(new) - median(old)) - maybe_print('%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff)) - s = speedup.speedup(new, old) + mdn_diff = abs(_median(new) - _median(old)) + _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % + (f, new_name, new, old_name, old, mdn_diff)) + s = bm_speedup.speedup(new, old) if abs(s) > 3 and mdn_diff > 0.5: self.final[f] = '%+d%%' % s return self.final.keys() @@ -103,21 +104,21 @@ class Benchmark: def row(self, flds): return [self.final[f] if f in self.final else '' for f in flds] -def read_json(filename): +def _read_json(filename): try: with open(filename) as f: return json.loads(f.read()) except ValueError, e: return None -def finalize(bms, loops, track): +def diff(bms, loops, track, old, new): benchmarks = collections.defaultdict(Benchmark) for bm in bms: for loop in range(0, loops): - js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop)) - js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop)) - js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop)) - js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop)) + js_new_ctr = _read_json('%s.counters.%s.%d.json' % (bm, new, loop)) + js_new_opt = _read_json('%s.opt.%s.%d.json' % (bm, new, loop)) + js_old_ctr = _read_json('%s.counters.%s.%d.json' % (bm, old, loop)) + js_old_opt = _read_json('%s.opt.%s.%d.json' % (bm, old, loop)) if js_new_ctr: for row in bm_json.expand_json(js_new_ctr, js_new_opt): @@ -132,8 +133,8 @@ def finalize(bms, loops, track): really_interesting = set() for name, bm in benchmarks.items(): - maybe_print(name) - really_interesting.update(bm.process(track)) + _maybe_print(name) + really_interesting.update(bm.process(track, new, old)) fields = [f for f in track if f in really_interesting] headers = ['Benchmark'] + fields @@ -148,6 +149,6 @@ def finalize(bms, loops, track): if __name__ == '__main__': args = _args() - print finalize(args.benchmarks, args.loops, args.track) + print diff(args.benchmarks, args.loops, args.track, args.old, args.new) diff --git a/tools/profiling/microbenchmarks/bm_main.py b/tools/profiling/microbenchmarks/bm_main.py new file mode 100755 index 00000000000..1a46b170155 --- /dev/null +++ b/tools/profiling/microbenchmarks/bm_main.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python2.7 +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" Runs the entire bm_*.py pipeline, and possible comments on the PR """ + +import bm_constants +import bm_build +import bm_run +import bm_diff + +import sys +import os +import argparse +import multiprocessing +import subprocess + +sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) +import comment_on_pr + +def _args(): + argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') + argp.add_argument('-t', '--track', + choices=sorted(bm_constants._INTERESTING), + nargs='+', + default=sorted(bm_constants._INTERESTING), + help='Which metrics to track') + argp.add_argument('-b', '--benchmarks', nargs='+', choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, default=bm_constants._AVAILABLE_BENCHMARK_TESTS) + argp.add_argument('-d', '--diff_base', type=str) + argp.add_argument('-r', '--repetitions', type=int, default=1) + argp.add_argument('-l', '--loops', type=int, default=20) + argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) + args = argp.parse_args() + assert args.diff_base + return args + + +def eintr_be_gone(fn): + """Run fn until it doesn't stop because of EINTR""" + def inner(*args): + while True: + try: + return fn(*args) + except IOError, e: + if e.errno != errno.EINTR: + raise + return inner + +def main(args): + + bm_build.build('new', args.benchmarks, args.jobs) + + where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_base]) + try: + bm_build.build('old', args.benchmarks, args.jobs) + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) + + bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions) + bm_run.run('old', args.benchmarks, args.jobs, args.loops, args.repetitions) + + diff = bm_diff.diff(args.benchmarks, args.loops, args.track, 'old', 'new') + if diff: + text = 'Performance differences noted:\n' + diff + else: + text = 'No significant performance differences' + print text + comment_on_pr.comment_on_pr('```\n%s\n```' % text) + +if __name__ == '__main__': + args = _args() + main(args) diff --git a/tools/profiling/microbenchmarks/bm_run.py b/tools/profiling/microbenchmarks/bm_run.py index 458e3194403..a476a7e19fb 100755 --- a/tools/profiling/microbenchmarks/bm_run.py +++ b/tools/profiling/microbenchmarks/bm_run.py @@ -51,27 +51,27 @@ def _args(): argp.add_argument('-l', '--loops', type=int, default=20) return argp.parse_args() -def _collect_bm_data(bm, cfg, name, reps, idx): +def _collect_bm_data(bm, cfg, name, reps, idx, loops): cmd = ['bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, name, idx), '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (reps) ] - return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, name, idx+1, args.loops), + return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, name, idx+1, loops), verbose_success=True, timeout_seconds=None) -def _run_bms(benchmarks, name, loops, reps): - jobs = [] +def run(name, benchmarks, jobs, loops, reps): + jobs_list = [] for loop in range(0, loops): - jobs.extend(x for x in itertools.chain( - (_collect_bm_data(bm, 'opt', name, reps, loop) for bm in benchmarks), - (_collect_bm_data(bm, 'counters', name, reps, loop) for bm in benchmarks), + jobs_list.extend(x for x in itertools.chain( + (_collect_bm_data(bm, 'opt', name, reps, loop, loops) for bm in benchmarks), + (_collect_bm_data(bm, 'counters', name, reps, loop, loops) for bm in benchmarks), )) - random.shuffle(jobs, random.SystemRandom().random) + random.shuffle(jobs_list, random.SystemRandom().random) - jobset.run(jobs, maxjobs=args.jobs) + jobset.run(jobs_list, maxjobs=jobs) if __name__ == '__main__': args = _args() assert args.name - _run_bms(args.benchmarks, args.name, args.loops, args.repetitions) + run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions)