Merge pull request #9943 from ctiller/diff_perf

Add ability to diff microbenchmark results between git revs

tools/run_tests/run_microbenchmark.py

@@ -170,12 +170,12 @@ def collect_perf(bm_name, args):
     jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
     jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
 
-def run_summary(cfg):
+def run_summary(bm_name, cfg, base_json_name):
   subprocess.check_call(
       ['make', bm_name,
        'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])
   cmd = ['bins/%s/%s' % (cfg, bm_name),
-         '--benchmark_out=out.%s.json' % cfg,
+         '--benchmark_out=%s.%s.json' % (base_json_name, cfg),
          '--benchmark_out_format=json']
   if args.summary_time is not None:
     cmd += ['--benchmark_min_time=%d' % args.summary_time]
@@ -183,9 +183,9 @@ def run_summary(cfg):
 
 def collect_summary(bm_name, args):
   heading('Summary: %s [no counters]' % bm_name)
-  text(run_summary('opt'))
+  text(run_summary(bm_name, 'opt', 'out'))
   heading('Summary: %s [with counters]' % bm_name)
-  text(run_summary('counters'))
+  text(run_summary(bm_name, 'counters', 'out'))
   if args.bigquery_upload:
     with open('out.csv', 'w') as f:
       f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.counters.json', 'out.opt.json']))
@@ -200,7 +200,7 @@ collectors = {
 argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
 argp.add_argument('-c', '--collect',
                   choices=sorted(collectors.keys()),
-                  nargs='+',
+                  nargs='*',
                   default=sorted(collectors.keys()),
                   help='Which collectors should be run against each benchmark')
 argp.add_argument('-b', '--benchmarks',
@@ -214,6 +214,10 @@ argp.add_argument('-b', '--benchmarks',
                   nargs='+',
                   type=str,
                   help='Which microbenchmarks should be run')
+argp.add_argument('--diff_perf',
+                  default=None,
+                  type=str,
+                  help='Diff microbenchmarks against this git revision')
 argp.add_argument('--bigquery_upload',
                   default=False,
                   action='store_const',
@@ -228,6 +232,26 @@ args = argp.parse_args()
 for bm_name in args.benchmarks:
   for collect in args.collect:
     collectors[collect](bm_name, args)
+if args.diff_perf:
+  for bm_name in args.benchmarks:
+    run_summary(bm_name, 'opt', '%s.new' % bm_name)
+  where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
+  subprocess.check_call(['git', 'checkout', args.diff_perf])
+  comparables = []
+  subprocess.check_call(['make', 'clean'])
+  try:
+    for bm_name in args.benchmarks:
+      try:
+        run_summary(bm_name, 'opt', '%s.old' % bm_name)
+        comparables.append(bm_name)
+      except subprocess.CalledProcessError, e:
+        pass
+  finally:
+    subprocess.check_call(['git', 'checkout', where_am_i])
+  for bm_name in comparables:
+    subprocess.check_call(['third_party/benchmark/tools/compare_bench.py',
+                          '%s.new.opt.json' % bm_name,
+                          '%s.old.opt.json' % bm_name])
 
 index_html += "</body>\n</html>\n"
 with open('reports/index.html', 'w') as f: