Shuffle new and old jobs together

tools/profiling/microbenchmarks/bm_diff/bm_main.py

@@ -23,6 +23,7 @@ import bm_diff
 
 import sys
 import os
+import random
 import argparse
 import multiprocessing
 import subprocess
@@ -32,6 +33,12 @@ sys.path.append(
     os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
 import comment_on_pr
 
+sys.path.append(
+  os.path.join(
+    os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
+    'python_utils'))
+import jobset
+
 
 def _args():
   argp = argparse.ArgumentParser(
@@ -125,8 +132,13 @@ def main(args):
       subprocess.check_call(['git', 'checkout', where_am_i])
       subprocess.check_call(['git', 'submodule', 'update'])
 
-  bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.regex, args.counters)
-  bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.regex, args.counters)
+  jobs_list = []
+  jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, args.regex, args.counters)
+  jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, args.regex, args.counters)
+
+  # shuffle all jobs to eliminate noise from GCE CPU drift
+  random.shuffle(jobs_list, random.SystemRandom().random)
+  jobset.run(jobs_list, maxjobs=args.jobs)
 
   diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex, args.track, old,
                 'new', args.counters)

tools/profiling/microbenchmarks/bm_diff/bm_run.py

@@ -99,7 +99,7 @@ def _collect_bm_data(bm, cfg, name, regex, idx, loops):
   return jobs_list
 
 
-def run(name, benchmarks, jobs, loops, regex, counters):
+def create_jobs(name, benchmarks, loops, regex, counters):
   jobs_list = []
   for loop in range(0, loops):
     for bm in benchmarks:
@@ -108,9 +108,11 @@ def run(name, benchmarks, jobs, loops, regex, counters):
         jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop,
                         loops)
   random.shuffle(jobs_list, random.SystemRandom().random)
-  jobset.run(jobs_list, maxjobs=jobs)
+  return jobs_list
 
 
 if __name__ == '__main__':
   args = _args()
-  run(args.name, args.benchmarks, args.jobs, args.loops, args.regex, args.counters)
+  jobs_list = create_jobs(args.name, args.benchmarks, args.loops, 
+                          args.regex, args.counters)
+  jobset.run(jobs_list, maxjobs=args.jobs)