diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 7b88512ee05..d3f23ddd796 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -20,4 +20,4 @@ set -ex cd $(dirname $0)/../.. tools/run_tests/start_port_server.py -tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls cli_stream_stalls svr_transport_stalls svr_stream_stalls +tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls cli_stream_stalls svr_transport_stalls svr_stream_stalls --no-counters diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_build.py b/tools/profiling/microbenchmarks/bm_diff/bm_build.py index 650ccdc2b21..c172de2af76 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_build.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_build.py @@ -46,6 +46,12 @@ def _args(): type=str, help='Unique name of this build. To be used as a handle to pass to the other bm* scripts' ) + argp.add_argument( + '-c', + '--counters', + type=bool, + default=True, + help='Whether or not to run and diff a counters build') args = argp.parse_args() assert args.name return args @@ -55,16 +61,18 @@ def _make_cmd(cfg, benchmarks, jobs): return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs] -def build(name, benchmarks, jobs): +def build(name, benchmarks, jobs, counters): shutil.rmtree('bm_diff_%s' % name, ignore_errors=True) subprocess.check_call(['git', 'submodule', 'update']) try: subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) - subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) + if counters: + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) except subprocess.CalledProcessError, e: subprocess.check_call(['make', 'clean']) subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) - subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) + if counters: + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) os.rename( 'bins', 'bm_diff_%s' % name,) @@ -72,4 +80,4 @@ def build(name, benchmarks, jobs): if __name__ == '__main__': args = _args() - build(args.name, args.benchmarks, args.jobs) + build(args.name, args.benchmarks, args.jobs, args.counters) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py index b8e803749a2..6bb53e9676c 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py @@ -67,6 +67,12 @@ def _args(): default=20, help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py' ) + argp.add_argument( + '-c', + '--counters', + type=bool, + default=True, + help='Whether or not to run and diff a counters build') argp.add_argument('-n', '--new', type=str, help='New benchmark name') argp.add_argument('-o', '--old', type=str, help='Old benchmark name') argp.add_argument( @@ -121,7 +127,8 @@ def _read_json(filename, badjson_files, nonexistant_files): stripped = ".".join(filename.split(".")[:-2]) try: with open(filename) as f: - return json.loads(f.read()) + r = f.read(); + return json.loads(r) except IOError, e: if stripped in nonexistant_files: nonexistant_files[stripped] += 1 @@ -129,14 +136,17 @@ def _read_json(filename, badjson_files, nonexistant_files): nonexistant_files[stripped] = 1 return None except ValueError, e: + print r if stripped in badjson_files: badjson_files[stripped] += 1 else: badjson_files[stripped] = 1 return None +def fmt_dict(d): + return ''.join([" " + k + ": " + str(d[k]) + "\n" for k in d]) -def diff(bms, loops, track, old, new): +def diff(bms, loops, track, old, new, counters): benchmarks = collections.defaultdict(Benchmark) badjson_files = {} @@ -148,18 +158,22 @@ def diff(bms, loops, track, old, new): '--benchmark_list_tests']).splitlines(): stripped_line = line.strip().replace("/", "_").replace( "<", "_").replace(">", "_").replace(", ", "_") - js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' % - (bm, stripped_line, new, loop), - badjson_files, nonexistant_files) js_new_opt = _read_json('%s.%s.opt.%s.%d.json' % (bm, stripped_line, new, loop), badjson_files, nonexistant_files) - js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' % - (bm, stripped_line, old, loop), - badjson_files, nonexistant_files) js_old_opt = _read_json('%s.%s.opt.%s.%d.json' % (bm, stripped_line, old, loop), badjson_files, nonexistant_files) + if counters: + js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' % + (bm, stripped_line, new, loop), + badjson_files, nonexistant_files) + js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' % + (bm, stripped_line, old, loop), + badjson_files, nonexistant_files) + else: + js_new_ctr = None + js_old_ctr = None if js_new_ctr: for row in bm_json.expand_json(js_new_ctr, js_new_opt): @@ -187,12 +201,12 @@ def diff(bms, loops, track, old, new): rows.append([name] + benchmarks[name].row(fields)) note = None if len(badjson_files): - note = 'Corrupt JSON data (indicates timeout or crash) = %s' % str(badjson_files) + note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(badjson_files) if len(nonexistant_files): if note: - note += '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files) + note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files) else: - note = '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files) + note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files) if rows: return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note else: @@ -202,5 +216,5 @@ def diff(bms, loops, track, old, new): if __name__ == '__main__': args = _args() diff, note = diff(args.benchmarks, args.loops, args.track, args.old, - args.new) + args.new, args.counters) print('%s\n%s' % (note, diff if diff else "No performance differences")) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py index 8a54f198ab2..51208cb965a 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_main.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py @@ -80,6 +80,9 @@ def _args(): type=int, default=multiprocessing.cpu_count(), help='Number of CPUs to use') + argp.add_argument('--counters', dest='counters', action='store_true') + argp.add_argument('--no-counters', dest='counters', action='store_false') + argp.set_defaults(counters=True) args = argp.parse_args() assert args.diff_base or args.old, "One of diff_base or old must be set!" if args.loops < 3: @@ -103,7 +106,7 @@ def eintr_be_gone(fn): def main(args): - bm_build.build('new', args.benchmarks, args.jobs) + bm_build.build('new', args.benchmarks, args.jobs, args.counters) old = args.old if args.diff_base: @@ -112,16 +115,16 @@ def main(args): ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) try: - bm_build.build('old', args.benchmarks, args.jobs) + bm_build.build(old, args.benchmarks, args.jobs, args.counters) finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) - bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions) - bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions) + bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters) + bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters) diff, note = bm_diff.diff(args.benchmarks, args.loops, args.track, old, - 'new') + 'new', args.counters) if diff: text = 'Performance differences noted:\n' + diff else: diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_run.py b/tools/profiling/microbenchmarks/bm_diff/bm_run.py index 3457af916b4..1b5cae0211a 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_run.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_run.py @@ -67,6 +67,12 @@ def _args(): default=20, help='Number of times to loops the benchmarks. More loops cuts down on noise' ) + argp.add_argument( + '-c', + '--counters', + type=bool, + default=True, + help='Whether or not to run and diff a counters build') args = argp.parse_args() assert args.name if args.loops < 3: @@ -93,21 +99,22 @@ def _collect_bm_data(bm, cfg, name, reps, idx, loops): shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1, loops), verbose_success=True, - timeout_seconds=60 * 2)) + timeout_seconds=60 * 60)) # one hour return jobs_list -def run(name, benchmarks, jobs, loops, reps): +def run(name, benchmarks, jobs, loops, reps, counters): jobs_list = [] for loop in range(0, loops): for bm in benchmarks: jobs_list += _collect_bm_data(bm, 'opt', name, reps, loop, loops) - jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop, - loops) + if counters: + jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop, + loops) random.shuffle(jobs_list, random.SystemRandom().random) jobset.run(jobs_list, maxjobs=jobs) if __name__ == '__main__': args = _args() - run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions) + run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)