adding speedup

tools/profiling/microbenchmarks/bm_diff.py

@@ -43,6 +43,7 @@ sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_test
 import comment_on_pr
 import jobset
 import itertools
+import speedup
 
 def changed_ratio(n, o):
   if float(o) <= .0001: o = 0
@@ -180,16 +181,9 @@ class Benchmark:
       new = self.samples[True][f]
       old = self.samples[False][f]
       if not new or not old: continue
-      p = stats.ttest_ind(new, old)[1]
-      new_mdn = median(new)
-      old_mdn = median(old)
-      delta = new_mdn - old_mdn
-      ratio = changed_ratio(new_mdn, old_mdn)
-      print '%s: new=%r old=%r new_mdn=%f old_mdn=%f delta=%f(%f:%f) ratio=%f(%f:%f) p=%f' % (
-      f, new, old, new_mdn, old_mdn, delta, abs(delta), _INTERESTING[f]['abs_diff'], ratio, abs(ratio), _INTERESTING[f]['pct_diff']/100.0, p
-      )
-      if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']/100.0:
-        self.final[f] = delta
+      s = speedup.speedup(new, old)
+      if s:
+        self.final[f] = '%d%%' % s
     return self.final.keys()
 
   def skip(self):
@@ -226,7 +220,7 @@ really_interesting = set()
 for name, bm in benchmarks.items():
   print name
   really_interesting.update(bm.process())
-fields = [f for f in args.track if f in args.track]
+fields = [f for f in args.track if f in really_interesting]
 
 headers = ['Benchmark'] + fields
 rows = []

tools/profiling/microbenchmarks/speedup.py

@@ -29,31 +29,37 @@
 
 from scipy import stats
 
-new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0]
-old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0]
-
 _THRESHOLD = 0.01
 
 def scale(a, mul):
   return [x*mul for x in a]
 
 def cmp(a, b):
-  return stats.mannwhitneyu(a, b, True, 'two-sided')
+  return stats.ttest_ind(a, b)
 
 def speedup(new, old):
   s0, p0 = cmp(new, old)
-  print s0, p0
   if s0 == 0: return 0
-  if p0 >= _THRESHOLD: return 0
+  if p0 > _THRESHOLD: return 0
   if s0 < 0:
-    pass
+    pct = 1
+    while True:
+      sp, pp = cmp(new, scale(old, 1 - pct/100.0))
+      if sp > 0: break
+      if pp > _THRESHOLD: break
+      pct += 1
+    return -(pct - 1)
   else:
     pct = 1
     while True:
-      sp, pp = cmp(new, scale(old, 1 + pct / 100.0))
-      if sp <= 0: break
-      if pp >= _THRESHOLD: break
+      sp, pp = cmp(new, scale(old, 1 + pct/100.0))
+      if sp < 0: break
+      if pp > _THRESHOLD: break
       pct += 1
     return pct - 1
 
-print speedup(new, old)
+if __name__ == "__main__":
+  new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0]
+  old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0]
+  print speedup(new, old)
+  print speedup(old, new)