From 0ae9959a1c93bedeb6f8ac009cb338eeee08a8ba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 1 Apr 2017 21:57:42 -0700 Subject: [PATCH 01/32] Revert "Revert "Add selected fullstack benchmarks to pr suite"" --- tools/jenkins/run_performance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index dee033a951a..544e31dcbdb 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -32,7 +32,7 @@ set -ex # List of benchmarks that provide good signal for analyzing performance changes in pull requests -BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" # Enter the gRPC repo root cd $(dirname $0)/../.. From f69317cf03cd69475e65e1e5ca2ef751947f3c12 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 2 Apr 2017 16:03:43 -0700 Subject: [PATCH 02/32] Increase reps to decrease noise --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 2337f2b282e..5350b5739f3 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -101,7 +101,7 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=4) +argp.add_argument('-r', '--repetitions', type=int, default=7) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) args = argp.parse_args() From 691eefbe0ff634fe49aacf06efccdd7e84864dfc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 08:06:32 -0700 Subject: [PATCH 03/32] Parallel bm_diff --- tools/profiling/microbenchmarks/bm_diff.py | 33 +++++++++++----------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 5350b5739f3..1602e83fcab 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +args.benchmarks#!/usr/bin/env python2.7 # Copyright 2017, Google Inc. # All rights reserved. # @@ -41,6 +41,7 @@ import pipes import os sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr +import jobset def changed_ratio(n, o): if float(o) <= .0001: o = 0 @@ -101,8 +102,9 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=7) +argp.add_argument('-r', '--repetitions', type=int, default=30) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) +argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() assert args.diff_base @@ -117,7 +119,7 @@ def avg(lst): def make_cmd(cfg): return ['make'] + args.benchmarks + [ - 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()] + 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs] def build(): subprocess.check_call(['git', 'submodule', 'update']) @@ -135,27 +137,24 @@ def collect1(bm, cfg, ver): '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) ] - print cmd - subprocess.check_call(cmd) + return jobset.JobSpec(cmd, shortname='%s %s %s' % (bm, cfg, ver), + verbose_success=True) build() -for bm in args.benchmarks: - collect1(bm, 'opt', 'new') - collect1(bm, 'counters', 'new') +jobset.run(itertools.chain( + (collect1(bm, 'opt', 'new') for bm in args.benchmarks), + (collect1(bm, 'counters', 'new') for bm in args.benchmarks), +), maxjobs=args.jobs) where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) try: build() - comparables = [] - for bm in args.benchmarks: - try: - collect1(bm, 'opt', 'old') - collect1(bm, 'counters', 'old') - comparables.append(bm) - except subprocess.CalledProcessError, e: - pass + jobset.run(itertools.chain( + (collect1(bm, 'opt', 'new') for bm in args.benchmarks), + (collect1(bm, 'counters', 'new') for bm in args.benchmarks), + ), maxjobs=args.jobs) finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) @@ -201,7 +200,7 @@ class Benchmark: benchmarks = collections.defaultdict(Benchmark) -for bm in comparables: +for bm in args.benchmarks: with open('%s.counters.new.json' % bm) as f: js_new_ctr = json.loads(f.read()) with open('%s.opt.new.json' % bm) as f: From cc89f7801bd1c6822e1805525073a69cf070320c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 08:08:25 -0700 Subject: [PATCH 04/32] Fix typo --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 1602e83fcab..82fbe084977 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -1,4 +1,4 @@ -args.benchmarks#!/usr/bin/env python2.7 +#!/usr/bin/env python2.7 # Copyright 2017, Google Inc. # All rights reserved. # From 2fbbcd1fd49203347b668bc2b896d8cbaab158fd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 08:29:08 -0700 Subject: [PATCH 05/32] Missing import --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 82fbe084977..84712e7e0dc 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -42,6 +42,7 @@ import os sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr import jobset +import itertools def changed_ratio(n, o): if float(o) <= .0001: o = 0 From 942324235cc03dd3437df8c282796f5711bfa131 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 10:44:03 -0700 Subject: [PATCH 06/32] s/new/old --- tools/profiling/microbenchmarks/bm_diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 84712e7e0dc..aa4caca43d7 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -153,8 +153,8 @@ subprocess.check_call(['git', 'checkout', args.diff_base]) try: build() jobset.run(itertools.chain( - (collect1(bm, 'opt', 'new') for bm in args.benchmarks), - (collect1(bm, 'counters', 'new') for bm in args.benchmarks), + (collect1(bm, 'opt', 'old') for bm in args.benchmarks), + (collect1(bm, 'counters', 'old') for bm in args.benchmarks), ), maxjobs=args.jobs) finally: subprocess.check_call(['git', 'checkout', where_am_i]) From 7e313858636917ceba4b37c673b8676afed75544 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 15:14:08 -0700 Subject: [PATCH 07/32] Remove timeout --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index aa4caca43d7..f6a944babc8 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -139,7 +139,7 @@ def collect1(bm, cfg, ver): '--benchmark_repetitions=%d' % (args.repetitions) ] return jobset.JobSpec(cmd, shortname='%s %s %s' % (bm, cfg, ver), - verbose_success=True) + verbose_success=True, timeout_seconds=None) build() jobset.run(itertools.chain( From a3589c5411186f0a29b661003eaa641d1a63af37 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 4 Apr 2017 21:09:08 -0700 Subject: [PATCH 08/32] Starting speedup calculation --- tools/profiling/microbenchmarks/speedup.py | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tools/profiling/microbenchmarks/speedup.py diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py new file mode 100644 index 00000000000..09222580ef9 --- /dev/null +++ b/tools/profiling/microbenchmarks/speedup.py @@ -0,0 +1,59 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from scipy import stats + +new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0] +old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0] + +_THRESHOLD = 0.01 + +def scale(a, mul): + return [x*mul for x in a] + +def cmp(a, b): + return stats.mannwhitneyu(a, b, True, 'two-sided') + +def speedup(new, old): + s0, p0 = cmp(new, old) + print s0, p0 + if s0 == 0: return 0 + if p0 >= _THRESHOLD: return 0 + if s0 < 0: + pass + else: + pct = 1 + while True: + sp, pp = cmp(new, scale(old, 1 + pct / 100.0)) + if sp <= 0: break + if pp >= _THRESHOLD: break + pct += 1 + return pct - 1 + +print speedup(new, old) From 125b04fcedd572fc171abe39cf181bafc8d08361 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 4 Apr 2017 21:31:44 -0700 Subject: [PATCH 09/32] adding speedup --- tools/profiling/microbenchmarks/bm_diff.py | 16 ++++--------- tools/profiling/microbenchmarks/speedup.py | 28 +++++++++++++--------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index f6a944babc8..5c8b93d471c 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -43,6 +43,7 @@ sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_test import comment_on_pr import jobset import itertools +import speedup def changed_ratio(n, o): if float(o) <= .0001: o = 0 @@ -180,16 +181,9 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue - p = stats.ttest_ind(new, old)[1] - new_mdn = median(new) - old_mdn = median(old) - delta = new_mdn - old_mdn - ratio = changed_ratio(new_mdn, old_mdn) - print '%s: new=%r old=%r new_mdn=%f old_mdn=%f delta=%f(%f:%f) ratio=%f(%f:%f) p=%f' % ( - f, new, old, new_mdn, old_mdn, delta, abs(delta), _INTERESTING[f]['abs_diff'], ratio, abs(ratio), _INTERESTING[f]['pct_diff']/100.0, p - ) - if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']/100.0: - self.final[f] = delta + s = speedup.speedup(new, old) + if s: + self.final[f] = '%d%%' % s return self.final.keys() def skip(self): @@ -226,7 +220,7 @@ really_interesting = set() for name, bm in benchmarks.items(): print name really_interesting.update(bm.process()) -fields = [f for f in args.track if f in args.track] +fields = [f for f in args.track if f in really_interesting] headers = ['Benchmark'] + fields rows = [] diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index 09222580ef9..9b022da0c46 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -29,31 +29,37 @@ from scipy import stats -new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0] -old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0] - _THRESHOLD = 0.01 def scale(a, mul): return [x*mul for x in a] def cmp(a, b): - return stats.mannwhitneyu(a, b, True, 'two-sided') + return stats.ttest_ind(a, b) def speedup(new, old): s0, p0 = cmp(new, old) - print s0, p0 if s0 == 0: return 0 - if p0 >= _THRESHOLD: return 0 + if p0 > _THRESHOLD: return 0 if s0 < 0: - pass + pct = 1 + while True: + sp, pp = cmp(new, scale(old, 1 - pct/100.0)) + if sp > 0: break + if pp > _THRESHOLD: break + pct += 1 + return -(pct - 1) else: pct = 1 while True: - sp, pp = cmp(new, scale(old, 1 + pct / 100.0)) - if sp <= 0: break - if pp >= _THRESHOLD: break + sp, pp = cmp(new, scale(old, 1 + pct/100.0)) + if sp < 0: break + if pp > _THRESHOLD: break pct += 1 return pct - 1 -print speedup(new, old) +if __name__ == "__main__": + new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0] + old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0] + print speedup(new, old) + print speedup(old, new) From e749e34f07be72b9f804039d1210b4ca5b045026 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 06:32:47 -0700 Subject: [PATCH 10/32] Provide bounds on speedup --- tools/profiling/microbenchmarks/speedup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index 9b022da0c46..ac268fb6790 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -43,7 +43,7 @@ def speedup(new, old): if p0 > _THRESHOLD: return 0 if s0 < 0: pct = 1 - while True: + while pct < 101: sp, pp = cmp(new, scale(old, 1 - pct/100.0)) if sp > 0: break if pp > _THRESHOLD: break @@ -51,7 +51,7 @@ def speedup(new, old): return -(pct - 1) else: pct = 1 - while True: + while pct < 101: sp, pp = cmp(new, scale(old, 1 + pct/100.0)) if sp < 0: break if pp > _THRESHOLD: break From d6d2da11492f52a4170be34c826f6adb8ca837d1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 06:34:07 -0700 Subject: [PATCH 11/32] Handle nans --- tools/profiling/microbenchmarks/speedup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index ac268fb6790..a06f2a91e1d 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -28,6 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from scipy import stats +import math _THRESHOLD = 0.01 @@ -39,6 +40,7 @@ def cmp(a, b): def speedup(new, old): s0, p0 = cmp(new, old) + if math.isnan(p0) return 0 if s0 == 0: return 0 if p0 > _THRESHOLD: return 0 if s0 < 0: From b1d3dcdb7db540b7dfa2efad269cd56738ff7e79 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 06:35:49 -0700 Subject: [PATCH 12/32] fix typo --- tools/profiling/microbenchmarks/speedup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index a06f2a91e1d..ff114539604 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -40,7 +40,7 @@ def cmp(a, b): def speedup(new, old): s0, p0 = cmp(new, old) - if math.isnan(p0) return 0 + if math.isnan(p0): return 0 if s0 == 0: return 0 if p0 > _THRESHOLD: return 0 if s0 < 0: From e5f18281755cbacf51613124c3f762ad0129b9d1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 09:08:37 -0700 Subject: [PATCH 13/32] Tweak parameters --- tools/profiling/microbenchmarks/bm_diff.py | 3 ++- tools/profiling/microbenchmarks/speedup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 5c8b93d471c..37d500d114f 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -181,9 +181,10 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue + print '%s: new=%r old=%r' % (f, new, old) s = speedup.speedup(new, old) if s: - self.final[f] = '%d%%' % s + self.final[f] = '%+d%%' % s return self.final.keys() def skip(self): diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index ff114539604..35d392a57db 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -30,7 +30,7 @@ from scipy import stats import math -_THRESHOLD = 0.01 +_THRESHOLD = 0.001 def scale(a, mul): return [x*mul for x in a] From 37f80e8a8a86612f44fd8e6cd3b15d73bc737404 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 12:18:30 -0700 Subject: [PATCH 14/32] Fix reuse bug? --- tools/profiling/microbenchmarks/bm_json.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index e885444f410..917269823d0 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -203,4 +203,5 @@ def expand_json(js, js2 = None): row['real_time'] = bm2['real_time'] row['iterations'] = bm2['iterations'] bm2['already_used'] = True + break yield row From 32873d171be6f874cd81312f6c951c80b975edc4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 12:19:51 -0700 Subject: [PATCH 15/32] accelerate --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 37d500d114f..09018594794 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -104,7 +104,7 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=30) +argp.add_argument('-r', '--repetitions', type=int, default=10) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() From b1c02cac82b9ec7988d60d53c3c2af8884c57f1a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 14:11:28 -0700 Subject: [PATCH 16/32] exclude small absolute changes --- tools/profiling/microbenchmarks/bm_diff.py | 25 +++------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 09018594794..3223cc1004d 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -63,26 +63,6 @@ def median(ary): def min_change(pct): return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 -nanos = { - 'abs_diff': 5, - 'pct_diff': 10, -} -counter = { - 'abs_diff': 0.5, - 'pct_diff': 10, -} - -_INTERESTING = { - 'cpu_time': nanos, - 'real_time': nanos, - 'locks_per_iteration': counter, - 'allocs_per_iteration': counter, - 'writes_per_iteration': counter, - 'atm_cas_per_iteration': counter, - 'atm_add_per_iteration': counter, -} - - _AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', 'bm_fullstack_streaming_ping_pong', 'bm_fullstack_streaming_pump', @@ -181,9 +161,10 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue - print '%s: new=%r old=%r' % (f, new, old) + mdn_diff = abs(median(new) - median(old)) + print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) s = speedup.speedup(new, old) - if s: + if s and mdn_diff > 0.5: self.final[f] = '%+d%%' % s return self.final.keys() From bdeb857ab4e2233bbb796378f7212eb52cf71bb3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 14:12:53 -0700 Subject: [PATCH 17/32] more samples --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 3223cc1004d..6c63f20b7df 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -84,7 +84,7 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=10) +argp.add_argument('-r', '--repetitions', type=int, default=30) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() From 7281d19de374852abf38521a20ef96b88e2916e9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 15:04:55 -0700 Subject: [PATCH 18/32] fix --- tools/profiling/microbenchmarks/bm_diff.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 6c63f20b7df..e4ef8df91c7 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -45,6 +45,16 @@ import jobset import itertools import speedup +_INTERESTING = ( + 'cpu_time', + 'real_time', + 'locks_per_iteration', + 'allocs_per_iteration', + 'writes_per_iteration', + 'atm_cas_per_iteration', + 'atm_add_per_iteration', +) + def changed_ratio(n, o): if float(o) <= .0001: o = 0 if float(n) <= .0001: n = 0 @@ -78,9 +88,9 @@ _AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') argp.add_argument('-t', '--track', - choices=sorted(_INTERESTING.keys()), + choices=sorted(_INTERESTING), nargs='+', - default=sorted(_INTERESTING.keys()), + default=sorted(_INTERESTING), help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) From f597c7ccdd1b808e131862094b3ef175202f088c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 14:39:32 -0700 Subject: [PATCH 19/32] Tighter threshold --- tools/profiling/microbenchmarks/speedup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index 35d392a57db..8f9023d2c89 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -30,7 +30,7 @@ from scipy import stats import math -_THRESHOLD = 0.001 +_THRESHOLD = 0.0001 def scale(a, mul): return [x*mul for x in a] From 2d7d5be813a53fc32046f338709b8c32d6bdd1fe Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 06:54:40 -0700 Subject: [PATCH 20/32] Increase threshold --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index e4ef8df91c7..c3738605194 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -174,7 +174,7 @@ class Benchmark: mdn_diff = abs(median(new) - median(old)) print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) s = speedup.speedup(new, old) - if s and mdn_diff > 0.5: + if s and mdn_diff > 0.5 and abs(s) >= 10: self.final[f] = '%+d%%' % s return self.final.keys() From 67dd855337d890656b46c8a6247ee924850fa935 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 16:42:30 -0700 Subject: [PATCH 21/32] Combine multiple temporally disconnected runs --- tools/profiling/microbenchmarks/bm_diff.py | 79 +++++++++++----------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index c3738605194..5bc0eacd22c 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -94,7 +94,8 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=30) +argp.add_argument('-r', '--repetitions', type=int, default=5) +argp.add_argument('-l', '--loops', type=int, default=5) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() @@ -123,33 +124,34 @@ def build(): subprocess.check_call(make_cmd('opt')) subprocess.check_call(make_cmd('counters')) -def collect1(bm, cfg, ver): +def collect1(bm, cfg, ver, idx): cmd = ['bins/%s/%s' % (cfg, bm), - '--benchmark_out=%s.%s.%s.json' % (bm, cfg, ver), + '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx), '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) ] return jobset.JobSpec(cmd, shortname='%s %s %s' % (bm, cfg, ver), verbose_success=True, timeout_seconds=None) -build() -jobset.run(itertools.chain( - (collect1(bm, 'opt', 'new') for bm in args.benchmarks), - (collect1(bm, 'counters', 'new') for bm in args.benchmarks), -), maxjobs=args.jobs) - -where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() -subprocess.check_call(['git', 'checkout', args.diff_base]) - -try: +for loop in range(0, args.loops): build() jobset.run(itertools.chain( - (collect1(bm, 'opt', 'old') for bm in args.benchmarks), - (collect1(bm, 'counters', 'old') for bm in args.benchmarks), + (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), + (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), ), maxjobs=args.jobs) -finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - subprocess.check_call(['git', 'submodule', 'update']) + + where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_base]) + + try: + build() + jobset.run(itertools.chain( + (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), + (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), + ), maxjobs=args.jobs) + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) class Benchmark: @@ -174,7 +176,7 @@ class Benchmark: mdn_diff = abs(median(new) - median(old)) print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) s = speedup.speedup(new, old) - if s and mdn_diff > 0.5 and abs(s) >= 10: + if s and mdn_diff > 0.5: self.final[f] = '%+d%%' % s return self.final.keys() @@ -188,25 +190,26 @@ class Benchmark: benchmarks = collections.defaultdict(Benchmark) for bm in args.benchmarks: - with open('%s.counters.new.json' % bm) as f: - js_new_ctr = json.loads(f.read()) - with open('%s.opt.new.json' % bm) as f: - js_new_opt = json.loads(f.read()) - with open('%s.counters.old.json' % bm) as f: - js_old_ctr = json.loads(f.read()) - with open('%s.opt.old.json' % bm) as f: - js_old_opt = json.loads(f.read()) - - for row in bm_json.expand_json(js_new_ctr, js_new_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, True) - for row in bm_json.expand_json(js_old_ctr, js_old_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, False) + for loop in args.loops: + with open('%s.counters.new.%d.json' % (bm, loop)) as f: + js_new_ctr = json.loads(f.read()) + with open('%s.opt.new.%d.json' % (bm, loop)) as f: + js_new_opt = json.loads(f.read()) + with open('%s.counters.old.%d.json' % (bm, loop)) as f: + js_old_ctr = json.loads(f.read()) + with open('%s.opt.old.%d.json' % (bm, loop)) as f: + js_old_opt = json.loads(f.read()) + + for row in bm_json.expand_json(js_new_ctr, js_new_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, True) + for row in bm_json.expand_json(js_old_ctr, js_old_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, False) really_interesting = set() for name, bm in benchmarks.items(): From 59bf91c72e4f8d249a004e4ac7879bec27be8bfd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 08:43:51 -0700 Subject: [PATCH 22/32] tweaks, fix bug --- tools/profiling/microbenchmarks/bm_diff.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 5bc0eacd22c..4fa4869b642 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -94,8 +94,8 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=5) -argp.add_argument('-l', '--loops', type=int, default=5) +argp.add_argument('-r', '--repetitions', type=int, default=3) +argp.add_argument('-l', '--loops', type=int, default=7) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() @@ -190,7 +190,7 @@ class Benchmark: benchmarks = collections.defaultdict(Benchmark) for bm in args.benchmarks: - for loop in args.loops: + for loop in range(0, args.loops): with open('%s.counters.new.%d.json' % (bm, loop)) as f: js_new_ctr = json.loads(f.read()) with open('%s.opt.new.%d.json' % (bm, loop)) as f: From 964647729e22d3e2fddf11409191e61fe8d1a971 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 22:06:25 -0700 Subject: [PATCH 23/32] Update bm_diff.py --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 4fa4869b642..4f21a76c610 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -176,7 +176,7 @@ class Benchmark: mdn_diff = abs(median(new) - median(old)) print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) s = speedup.speedup(new, old) - if s and mdn_diff > 0.5: + if abs(s) > 3 and mdn_diff > 0.5: self.final[f] = '%+d%%' % s return self.final.keys() From 233ad390fa30ff90d927dd29f277a914a0cf0621 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:25:53 -0700 Subject: [PATCH 24/32] Accelerate by running more jobs in parallel, and only building twice --- tools/profiling/microbenchmarks/bm_diff.py | 41 ++++++++++++---------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 4f21a76c610..d1d49037cb3 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -44,6 +44,7 @@ import comment_on_pr import jobset import itertools import speedup +import random _INTERESTING = ( 'cpu_time', @@ -114,7 +115,7 @@ def make_cmd(cfg): return ['make'] + args.benchmarks + [ 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs] -def build(): +def build(dest): subprocess.check_call(['git', 'submodule', 'update']) try: subprocess.check_call(make_cmd('opt')) @@ -123,6 +124,7 @@ def build(): subprocess.check_call(['make', 'clean']) subprocess.check_call(make_cmd('opt')) subprocess.check_call(make_cmd('counters')) + os.rename('bin', dest) def collect1(bm, cfg, ver, idx): cmd = ['bins/%s/%s' % (cfg, bm), @@ -130,29 +132,30 @@ def collect1(bm, cfg, ver, idx): '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) ] - return jobset.JobSpec(cmd, shortname='%s %s %s' % (bm, cfg, ver), + return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, ver, idx+1, args.loops), verbose_success=True, timeout_seconds=None) -for loop in range(0, args.loops): - build() - jobset.run(itertools.chain( - (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), - (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), - ), maxjobs=args.jobs) +build('new') - where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() - subprocess.check_call(['git', 'checkout', args.diff_base]) +where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() +subprocess.check_call(['git', 'checkout', args.diff_base]) - try: - build() - jobset.run(itertools.chain( - (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), - (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), - ), maxjobs=args.jobs) - finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - subprocess.check_call(['git', 'submodule', 'update']) +try: + build('old') +finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) + +for loop in args.loops: + jobs.extend(x for x in itertools.chain( + (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), + (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), + (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), + (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), + )) +random.shuffle(jobs) +jobset.run(jobs, maxjobs=args.jobs) class Benchmark: From a00e51ccfee3ca6877ef568c8a5d65867ce974ca Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:27:39 -0700 Subject: [PATCH 25/32] Fix dir names --- tools/profiling/microbenchmarks/bm_diff.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index d1d49037cb3..634e2814466 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -124,10 +124,10 @@ def build(dest): subprocess.check_call(['make', 'clean']) subprocess.check_call(make_cmd('opt')) subprocess.check_call(make_cmd('counters')) - os.rename('bin', dest) + os.rename('bins', dest) def collect1(bm, cfg, ver, idx): - cmd = ['bins/%s/%s' % (cfg, bm), + cmd = ['%s/%s/%s' % (ver, cfg, bm), '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx), '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) @@ -139,7 +139,6 @@ build('new') where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) - try: build('old') finally: From 0553b9c75d87394ebddb53722e11acaec2305810 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:31:17 -0700 Subject: [PATCH 26/32] Properly clean --- tools/profiling/microbenchmarks/bm_diff.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 634e2814466..d22a97fd1ea 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -45,6 +45,7 @@ import jobset import itertools import speedup import random +import shutil _INTERESTING = ( 'cpu_time', @@ -116,6 +117,7 @@ def make_cmd(cfg): 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs] def build(dest): + shutil.rmtree('bm_diff_%s' % dest, ignore_errors=True) subprocess.check_call(['git', 'submodule', 'update']) try: subprocess.check_call(make_cmd('opt')) @@ -124,10 +126,10 @@ def build(dest): subprocess.check_call(['make', 'clean']) subprocess.check_call(make_cmd('opt')) subprocess.check_call(make_cmd('counters')) - os.rename('bins', dest) + os.rename('bins', 'bm_diff_%s' % dest) def collect1(bm, cfg, ver, idx): - cmd = ['%s/%s/%s' % (ver, cfg, bm), + cmd = ['bm_diff_%s/%s/%s' % (ver, cfg, bm), '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx), '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) From 0ebff8b2137f799b1beaebf89835b060c53461a3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:32:38 -0700 Subject: [PATCH 27/32] Iterate properly --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index d22a97fd1ea..07c145e4e48 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -147,7 +147,7 @@ finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) -for loop in args.loops: +for loop in range(0, args.loops): jobs.extend(x for x in itertools.chain( (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), From bc442e1ff0e2fd8107d41219c3c52472324efa77 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:33:55 -0700 Subject: [PATCH 28/32] Declare jobs --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 07c145e4e48..a6e77b375f1 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -147,6 +147,7 @@ finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) +jobs = [] for loop in range(0, args.loops): jobs.extend(x for x in itertools.chain( (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), From d71ba8392aadb43eb015270499c711a4221ddd95 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:36:11 -0700 Subject: [PATCH 29/32] Tweak defaults --- tools/profiling/microbenchmarks/bm_diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index a6e77b375f1..bd0da57598f 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -96,8 +96,8 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=3) -argp.add_argument('-l', '--loops', type=int, default=7) +argp.add_argument('-r', '--repetitions', type=int, default=1) +argp.add_argument('-l', '--loops', type=int, default=30) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() From 13c0e6e549175f29cb2d22bd3b2f7b00e5c2e94c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:40:04 -0700 Subject: [PATCH 30/32] Update .gitignore --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index ad3ec64ab5a..5e38f5fa01b 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,13 @@ gdb.txt # ctags file tags + +# perf data +perf.data +perf.data.old + +# bm_diff +bm_diff_new/ +bm_diff_old/ +bm_*.json + From 97e40da5a1219ccd838e0818d060b217c0830ae5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 09:04:07 -0700 Subject: [PATCH 31/32] Less samples, more random --- tools/profiling/microbenchmarks/bm_diff.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index bd0da57598f..3d4fa2a96a5 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -97,8 +97,7 @@ argp.add_argument('-t', '--track', argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) argp.add_argument('-r', '--repetitions', type=int, default=1) -argp.add_argument('-l', '--loops', type=int, default=30) -argp.add_argument('-p', '--p_threshold', type=float, default=0.01) +argp.add_argument('-l', '--loops', type=int, default=12) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() @@ -155,7 +154,7 @@ for loop in range(0, args.loops): (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), )) -random.shuffle(jobs) +random.shuffle(jobs, random.SystemRandom().random) jobset.run(jobs, maxjobs=args.jobs) From 349907b6459b639a937130f4de23e785cbf289d1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 13:00:47 -0700 Subject: [PATCH 32/32] Handle EINTR on reading file --- tools/profiling/microbenchmarks/bm_diff.py | 25 +++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 3d4fa2a96a5..ba0c225f6cc 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -46,6 +46,7 @@ import itertools import speedup import random import shutil +import errno _INTERESTING = ( 'cpu_time', @@ -191,18 +192,26 @@ class Benchmark: return [self.final[f] if f in self.final else '' for f in flds] +def read_file(filename): + while True: + try: + with open(filename) as f: + return f.read() + except IOError, e: + if e.errno != errno.EINTR: + raise + +def read_json(filename): + return json.loads(read_file(filename)) + benchmarks = collections.defaultdict(Benchmark) for bm in args.benchmarks: for loop in range(0, args.loops): - with open('%s.counters.new.%d.json' % (bm, loop)) as f: - js_new_ctr = json.loads(f.read()) - with open('%s.opt.new.%d.json' % (bm, loop)) as f: - js_new_opt = json.loads(f.read()) - with open('%s.counters.old.%d.json' % (bm, loop)) as f: - js_old_ctr = json.loads(f.read()) - with open('%s.opt.old.%d.json' % (bm, loop)) as f: - js_old_opt = json.loads(f.read()) + js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop)) + js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop)) + js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop)) + js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop)) for row in bm_json.expand_json(js_new_ctr, js_new_opt): print row