From 6a17fddc6996d555d508cbda50b990ca2bca3c82 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 08:21:10 -0700 Subject: [PATCH] Threshold % change also --- tools/profiling/microbenchmarks/bm_diff.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 43ba1b47e05..84217ec40ba 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -161,8 +161,11 @@ class Benchmark: old = self.samples[False][f] if not new or not old: continue p = stats.ttest_ind(new, old)[1] - delta = avg(new) - avg(old) - if p < args.p_threshold and abs(delta) > 0.1: + new_avg = avg(new) + old_avg = avg(old) + delta = new_avg - old_avg + ratio = changed_ratio(new_avg, old_avg) + if p < args.p_threshold and abs(delta) > 0.1 and abs(ratio) > 0.05: self.final[f] = delta return self.final.keys() @@ -210,4 +213,3 @@ else: text = 'No significant performance differences' comment_on_pr.comment_on_pr(text) print text -