#!/usr/bin/env python3 # # Copyright 2017 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Computes the diff between two bm runs and outputs significant results """ import argparse import collections import json import os import subprocess import sys sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..')) import bm_constants import bm_json import bm_speedup import tabulate verbose = False def _median(ary): assert (len(ary)) ary = sorted(ary) n = len(ary) if n % 2 == 0: return (ary[(n - 1) // 2] + ary[(n - 1) // 2 + 1]) / 2.0 else: return ary[n // 2] def _args(): argp = argparse.ArgumentParser( description='Perform diff on microbenchmarks') argp.add_argument('-t', '--track', choices=sorted(bm_constants._INTERESTING), nargs='+', default=sorted(bm_constants._INTERESTING), help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, default=bm_constants._AVAILABLE_BENCHMARK_TESTS, help='Which benchmarks to run') argp.add_argument( '-l', '--loops', type=int, default=20, help= 'Number of times to loops the benchmarks. Must match what was passed to bm_run.py' ) argp.add_argument('-r', '--regex', type=str, default="", help='Regex to filter benchmarks run') argp.add_argument('--counters', dest='counters', action='store_true') argp.add_argument('--no-counters', dest='counters', action='store_false') argp.set_defaults(counters=True) argp.add_argument('-n', '--new', type=str, help='New benchmark name') argp.add_argument('-o', '--old', type=str, help='Old benchmark name') argp.add_argument('-v', '--verbose', type=bool, help='Print details of before/after') args = argp.parse_args() global verbose if args.verbose: verbose = True assert args.new assert args.old return args def _maybe_print(str): if verbose: print(str) class Benchmark: def __init__(self): self.samples = { True: collections.defaultdict(list), False: collections.defaultdict(list) } self.final = {} self.speedup = {} def add_sample(self, track, data, new): for f in track: if f in data: self.samples[new][f].append(float(data[f])) def process(self, track, new_name, old_name): for f in sorted(track): new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue mdn_diff = abs(_median(new) - _median(old)) _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % (f, new_name, new, old_name, old, mdn_diff)) s = bm_speedup.speedup(new, old, 1e-5) self.speedup[f] = s if abs(s) > 3: if mdn_diff > 0.5: self.final[f] = '%+d%%' % s return self.final.keys() def skip(self): return not self.final def row(self, flds): return [self.final[f] if f in self.final else '' for f in flds] def speedup(self, name): if name in self.speedup: return self.speedup[name] return None def _read_json(filename, badjson_files, nonexistant_files): stripped = ".".join(filename.split(".")[:-2]) try: with open(filename) as f: r = f.read() return json.loads(r) except IOError as e: if stripped in nonexistant_files: nonexistant_files[stripped] += 1 else: nonexistant_files[stripped] = 1 return None except ValueError as e: print(r) if stripped in badjson_files: badjson_files[stripped] += 1 else: badjson_files[stripped] = 1 return None def fmt_dict(d): return ''.join([" " + k + ": " + str(d[k]) + "\n" for k in d]) def diff(bms, loops, regex, track, old, new, counters): benchmarks = collections.defaultdict(Benchmark) badjson_files = {} nonexistant_files = {} for bm in bms: for loop in range(0, loops): for line in subprocess.check_output([ 'bm_diff_%s/opt/%s' % (old, bm), '--benchmark_list_tests', '--benchmark_filter=%s' % regex ]).splitlines(): line = line.decode('UTF-8') stripped_line = line.strip().replace("/", "_").replace( "<", "_").replace(">", "_").replace(", ", "_") js_new_opt = _read_json( '%s.%s.opt.%s.%d.json' % (bm, stripped_line, new, loop), badjson_files, nonexistant_files) js_old_opt = _read_json( '%s.%s.opt.%s.%d.json' % (bm, stripped_line, old, loop), badjson_files, nonexistant_files) if counters: js_new_ctr = _read_json( '%s.%s.counters.%s.%d.json' % (bm, stripped_line, new, loop), badjson_files, nonexistant_files) js_old_ctr = _read_json( '%s.%s.counters.%s.%d.json' % (bm, stripped_line, old, loop), badjson_files, nonexistant_files) else: js_new_ctr = None js_old_ctr = None for row in bm_json.expand_json(js_new_ctr, js_new_opt): name = row['cpp_name'] if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(track, row, True) for row in bm_json.expand_json(js_old_ctr, js_old_opt): name = row['cpp_name'] if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(track, row, False) really_interesting = set() for name, bm in benchmarks.items(): _maybe_print(name) really_interesting.update(bm.process(track, new, old)) fields = [f for f in track if f in really_interesting] # figure out the significance of the changes... right now we take the 95%-ile # benchmark delta %-age, and then apply some hand chosen thresholds histogram = [] for bm in benchmarks.values(): if bm.skip(): continue d = bm.speedup('cpu_time') if d is None: continue histogram.append(d) histogram.sort() print("histogram of speedups: ", histogram) if len(histogram) == 0: significance = 0 else: delta = histogram[int(len(histogram) * 0.95)] mul = 1 if delta < 0: delta = -delta mul = -1 if delta < 2: significance = 0 elif delta < 5: significance = 1 elif delta < 10: significance = 2 else: significance = 3 significance *= mul headers = ['Benchmark'] + fields rows = [] for name in sorted(benchmarks.keys()): if benchmarks[name].skip(): continue rows.append([name] + benchmarks[name].row(fields)) note = None if len(badjson_files): note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict( badjson_files) if len(nonexistant_files): if note: note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict( nonexistant_files) else: note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict( nonexistant_files) if rows: return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note, significance else: return None, note, 0 if __name__ == '__main__': args = _args() diff, note = diff(args.benchmarks, args.loops, args.regex, args.track, args.old, args.new, args.counters) print('%s\n%s' % (note, diff if diff else "No performance differences"))