diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index f6a944babc8..5c8b93d471c 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -43,6 +43,7 @@ sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_test import comment_on_pr import jobset import itertools +import speedup def changed_ratio(n, o): if float(o) <= .0001: o = 0 @@ -180,16 +181,9 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue - p = stats.ttest_ind(new, old)[1] - new_mdn = median(new) - old_mdn = median(old) - delta = new_mdn - old_mdn - ratio = changed_ratio(new_mdn, old_mdn) - print '%s: new=%r old=%r new_mdn=%f old_mdn=%f delta=%f(%f:%f) ratio=%f(%f:%f) p=%f' % ( - f, new, old, new_mdn, old_mdn, delta, abs(delta), _INTERESTING[f]['abs_diff'], ratio, abs(ratio), _INTERESTING[f]['pct_diff']/100.0, p - ) - if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']/100.0: - self.final[f] = delta + s = speedup.speedup(new, old) + if s: + self.final[f] = '%d%%' % s return self.final.keys() def skip(self): @@ -226,7 +220,7 @@ really_interesting = set() for name, bm in benchmarks.items(): print name really_interesting.update(bm.process()) -fields = [f for f in args.track if f in args.track] +fields = [f for f in args.track if f in really_interesting] headers = ['Benchmark'] + fields rows = [] diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index 09222580ef9..9b022da0c46 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -29,31 +29,37 @@ from scipy import stats -new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0] -old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0] - _THRESHOLD = 0.01 def scale(a, mul): return [x*mul for x in a] def cmp(a, b): - return stats.mannwhitneyu(a, b, True, 'two-sided') + return stats.ttest_ind(a, b) def speedup(new, old): s0, p0 = cmp(new, old) - print s0, p0 if s0 == 0: return 0 - if p0 >= _THRESHOLD: return 0 + if p0 > _THRESHOLD: return 0 if s0 < 0: - pass + pct = 1 + while True: + sp, pp = cmp(new, scale(old, 1 - pct/100.0)) + if sp > 0: break + if pp > _THRESHOLD: break + pct += 1 + return -(pct - 1) else: pct = 1 while True: - sp, pp = cmp(new, scale(old, 1 + pct / 100.0)) - if sp <= 0: break - if pp >= _THRESHOLD: break + sp, pp = cmp(new, scale(old, 1 + pct/100.0)) + if sp < 0: break + if pp > _THRESHOLD: break pct += 1 return pct - 1 -print speedup(new, old) +if __name__ == "__main__": + new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0] + old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0] + print speedup(new, old) + print speedup(old, new)