Merge pull request #11467 from ncteisen/bm-trickle-diff

Add Trickle Diff Jenkins Job

tools/jenkins/run_performance.sh

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# This script is invoked by Jenkins and runs performance smoke test.
+# This script is invoked by Jenkins and runs a diff on the microbenchmarks
 set -ex
 
 # List of benchmarks that provide good signal for analyzing performance changes in pull requests

tools/jenkins/run_trickle_diff.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script is invoked by Jenkins and runs a diff on bm_fullstack_trickle
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../..
+
+tools/run_tests/start_port_server.py
+tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls cli_stream_stalls svr_transport_stalls svr_stream_stalls --no-counters --pr_comment_name trickle

tools/profiling/microbenchmarks/bm_diff/bm_build.py

@@ -46,6 +46,9 @@ def _args():
     type=str,
     help='Unique name of this build. To be used as a handle to pass to the other bm* scripts'
   )
+  argp.add_argument('--counters', dest='counters', action='store_true')
+  argp.add_argument('--no-counters', dest='counters', action='store_false')
+  argp.set_defaults(counters=True)
   args = argp.parse_args()
   assert args.name
   return args
@@ -55,16 +58,18 @@ def _make_cmd(cfg, benchmarks, jobs):
   return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs]
 
 
-def build(name, benchmarks, jobs):
+def build(name, benchmarks, jobs, counters):
   shutil.rmtree('bm_diff_%s' % name, ignore_errors=True)
   subprocess.check_call(['git', 'submodule', 'update'])
   try:
     subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
-    subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
+    if counters:
+      subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
   except subprocess.CalledProcessError, e:
     subprocess.check_call(['make', 'clean'])
     subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
-    subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
+    if counters:
+      subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
   os.rename(
     'bins',
     'bm_diff_%s' % name,)
@@ -72,4 +77,4 @@ def build(name, benchmarks, jobs):
 
 if __name__ == '__main__':
   args = _args()
-  build(args.name, args.benchmarks, args.jobs)
+  build(args.name, args.benchmarks, args.jobs, args.counters)

tools/profiling/microbenchmarks/bm_diff/bm_constants.py

@@ -26,4 +26,5 @@ _AVAILABLE_BENCHMARK_TESTS = [
 _INTERESTING = ('cpu_time', 'real_time', 'locks_per_iteration',
         'allocs_per_iteration', 'writes_per_iteration',
         'atm_cas_per_iteration', 'atm_add_per_iteration',
-        'nows_per_iteration',)
+        'nows_per_iteration', 'cli_transport_stalls', 'cli_stream_stalls',
+        'svr_transport_stalls', 'svr_stream_stalls',)

tools/profiling/microbenchmarks/bm_diff/bm_diff.py

@@ -67,6 +67,9 @@ def _args():
     default=20,
     help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py'
   )
+  argp.add_argument('--counters', dest='counters', action='store_true')
+  argp.add_argument('--no-counters', dest='counters', action='store_false')
+  argp.set_defaults(counters=True)
   argp.add_argument('-n', '--new', type=str, help='New benchmark name')
   argp.add_argument('-o', '--old', type=str, help='Old benchmark name')
   argp.add_argument(
@@ -121,7 +124,8 @@ def _read_json(filename, badjson_files, nonexistant_files):
   stripped = ".".join(filename.split(".")[:-2])
   try:
     with open(filename) as f:
-      return json.loads(f.read())
+      r = f.read();
+      return json.loads(r)
   except IOError, e:
     if stripped in nonexistant_files:
       nonexistant_files[stripped] += 1
@@ -129,14 +133,17 @@ def _read_json(filename, badjson_files, nonexistant_files):
       nonexistant_files[stripped] = 1
     return None
   except ValueError, e:
+    print r
     if stripped in badjson_files:
       badjson_files[stripped] += 1
     else:
       badjson_files[stripped] = 1
     return None
 
+def fmt_dict(d):
+  return ''.join(["    " + k + ": " + str(d[k]) + "\n" for k in d])
 
-def diff(bms, loops, track, old, new):
+def diff(bms, loops, track, old, new, counters):
   benchmarks = collections.defaultdict(Benchmark)
 
   badjson_files = {}
@@ -148,18 +155,22 @@ def diff(bms, loops, track, old, new):
          '--benchmark_list_tests']).splitlines():
         stripped_line = line.strip().replace("/", "_").replace(
           "<", "_").replace(">", "_").replace(", ", "_")
-        js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
-                    (bm, stripped_line, new, loop),
-                    badjson_files, nonexistant_files)
         js_new_opt = _read_json('%s.%s.opt.%s.%d.json' %
                     (bm, stripped_line, new, loop),
                     badjson_files, nonexistant_files)
-        js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
-                    (bm, stripped_line, old, loop),
-                    badjson_files, nonexistant_files)
         js_old_opt = _read_json('%s.%s.opt.%s.%d.json' %
                     (bm, stripped_line, old, loop),
                     badjson_files, nonexistant_files)
+        if counters:
+          js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+                      (bm, stripped_line, new, loop),
+                      badjson_files, nonexistant_files)
+          js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+                      (bm, stripped_line, old, loop),
+                      badjson_files, nonexistant_files)
+        else:
+          js_new_ctr = None
+          js_old_ctr = None
 
         if js_new_ctr:
           for row in bm_json.expand_json(js_new_ctr, js_new_opt):
@@ -187,12 +198,12 @@ def diff(bms, loops, track, old, new):
     rows.append([name] + benchmarks[name].row(fields))
   note = None
   if len(badjson_files):
-    note = 'Corrupt JSON data (indicates timeout or crash) = %s' % str(badjson_files)
+    note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(badjson_files)
   if len(nonexistant_files):
     if note:
-      note += '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files)
+      note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
     else:
-      note = '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files)
+      note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
   if rows:
     return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note
   else:
@@ -202,5 +213,5 @@ def diff(bms, loops, track, old, new):
 if __name__ == '__main__':
   args = _args()
   diff, note = diff(args.benchmarks, args.loops, args.track, args.old,
-            args.new)
+            args.new, args.counters)
   print('%s\n%s' % (note, diff if diff else "No performance differences"))

tools/profiling/microbenchmarks/bm_diff/bm_main.py

@@ -80,6 +80,14 @@ def _args():
     type=int,
     default=multiprocessing.cpu_count(),
     help='Number of CPUs to use')
+  argp.add_argument(
+    '--pr_comment_name',
+    type=str,
+    default="microbenchmarks",
+    help='Name that Jenkins will use to commen on the PR')
+  argp.add_argument('--counters', dest='counters', action='store_true')
+  argp.add_argument('--no-counters', dest='counters', action='store_false')
+  argp.set_defaults(counters=True)
   args = argp.parse_args()
   assert args.diff_base or args.old, "One of diff_base or old must be set!"
   if args.loops < 3:
@@ -103,7 +111,7 @@ def eintr_be_gone(fn):
 
 def main(args):
 
-  bm_build.build('new', args.benchmarks, args.jobs)
+  bm_build.build('new', args.benchmarks, args.jobs, args.counters)
 
   old = args.old
   if args.diff_base:
@@ -112,20 +120,20 @@ def main(args):
       ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
     subprocess.check_call(['git', 'checkout', args.diff_base])
     try:
-      bm_build.build('old', args.benchmarks, args.jobs)
+      bm_build.build(old, args.benchmarks, args.jobs, args.counters)
     finally:
       subprocess.check_call(['git', 'checkout', where_am_i])
       subprocess.check_call(['git', 'submodule', 'update'])
 
-  bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions)
-  bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions)
+  bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)
+  bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)
 
   diff, note = bm_diff.diff(args.benchmarks, args.loops, args.track, old,
-                'new')
+                'new', args.counters)
   if diff:
-    text = 'Performance differences noted:\n' + diff
+    text = '[%s] Performance differences noted:\n%s' % (args.pr_comment_name, diff)
   else:
-    text = 'No significant performance differences'
+    text = '[%s] No significant performance differences' % args.pr_comment_name
   if note:
     text = note + '\n\n' + text
   print('%s' % text)

tools/profiling/microbenchmarks/bm_diff/bm_run.py

@@ -67,6 +67,9 @@ def _args():
     default=20,
     help='Number of times to loops the benchmarks. More loops cuts down on noise'
   )
+  argp.add_argument('--counters', dest='counters', action='store_true')
+  argp.add_argument('--no-counters', dest='counters', action='store_false')
+  argp.set_defaults(counters=True)
   args = argp.parse_args()
   assert args.name
   if args.loops < 3:
@@ -93,21 +96,22 @@ def _collect_bm_data(bm, cfg, name, reps, idx, loops):
         shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1,
                          loops),
         verbose_success=True,
-        timeout_seconds=60 * 2))
+        timeout_seconds=60 * 60)) # one hour
   return jobs_list
 
 
-def run(name, benchmarks, jobs, loops, reps):
+def run(name, benchmarks, jobs, loops, reps, counters):
   jobs_list = []
   for loop in range(0, loops):
     for bm in benchmarks:
       jobs_list += _collect_bm_data(bm, 'opt', name, reps, loop, loops)
-      jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop,
-                      loops)
+      if counters:
+        jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop,
+                        loops)
   random.shuffle(jobs_list, random.SystemRandom().random)
   jobset.run(jobs_list, maxjobs=jobs)
 
 
 if __name__ == '__main__':
   args = _args()
-  run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions)
+  run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)