Chiebot-Mirror
/
grpc
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Disable counters for trickle, inc timeout

Browse Source
pull/11467/head
ncteisen 8 years ago
parent 12da4ccb52
commit 6dc4c7f5d9
5 changed files with 59 additions and 27 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      tools/jenkins/run_performance.sh
  2. 16
      tools/profiling/microbenchmarks/bm_diff/bm_build.py
  3. 38
      tools/profiling/microbenchmarks/bm_diff/bm_diff.py
  4. 13
      tools/profiling/microbenchmarks/bm_diff/bm_main.py
  5. 17
      tools/profiling/microbenchmarks/bm_diff/bm_run.py

2
tools/jenkins/run_performance.sh
Unescape View File

@ -20,4 +20,4 @@ set -ex
cd $(dirname $0)/../..
tools/run_tests/start_port_server.py
tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls cli_stream_stalls svr_transport_stalls svr_stream_stalls
tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls cli_stream_stalls svr_transport_stalls svr_stream_stalls --no-counters

16
tools/profiling/microbenchmarks/bm_diff/bm_build.py
Unescape View File

@ -46,6 +46,12 @@ def _args():
type=str,
help='Unique name of this build. To be used as a handle to pass to the other bm* scripts'
)
argp.add_argument(
'-c',
'--counters',
type=bool,
default=True,
help='Whether or not to run and diff a counters build')
args = argp.parse_args()
assert args.name
return args
@ -55,16 +61,18 @@ def _make_cmd(cfg, benchmarks, jobs):
return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs]
def build(name, benchmarks, jobs):
def build(name, benchmarks, jobs, counters):
shutil.rmtree('bm_diff_%s' % name, ignore_errors=True)
subprocess.check_call(['git', 'submodule', 'update'])
try:
subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
if counters:
subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
except subprocess.CalledProcessError, e:
subprocess.check_call(['make', 'clean'])
subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
if counters:
subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
os.rename(
'bins',
'bm_diff_%s' % name,)
@ -72,4 +80,4 @@ def build(name, benchmarks, jobs):
if __name__ == '__main__':
args = _args()
build(args.name, args.benchmarks, args.jobs)
build(args.name, args.benchmarks, args.jobs, args.counters)

38
tools/profiling/microbenchmarks/bm_diff/bm_diff.py
Unescape View File

@ -67,6 +67,12 @@ def _args():
default=20,
help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py'
)
argp.add_argument(
'-c',
'--counters',
type=bool,
default=True,
help='Whether or not to run and diff a counters build')
argp.add_argument('-n', '--new', type=str, help='New benchmark name')
argp.add_argument('-o', '--old', type=str, help='Old benchmark name')
argp.add_argument(
@ -121,7 +127,8 @@ def _read_json(filename, badjson_files, nonexistant_files):
stripped = ".".join(filename.split(".")[:-2])
try:
with open(filename) as f:
return json.loads(f.read())
r = f.read();
return json.loads(r)
except IOError, e:
if stripped in nonexistant_files:
nonexistant_files[stripped] += 1
@ -129,14 +136,17 @@ def _read_json(filename, badjson_files, nonexistant_files):
nonexistant_files[stripped] = 1
return None
except ValueError, e:
print r
if stripped in badjson_files:
badjson_files[stripped] += 1
else:
badjson_files[stripped] = 1
return None
def fmt_dict(d):
return ''.join([" " + k + ": " + str(d[k]) + "\n" for k in d])
def diff(bms, loops, track, old, new):
def diff(bms, loops, track, old, new, counters):
benchmarks = collections.defaultdict(Benchmark)
badjson_files = {}
@ -148,18 +158,22 @@ def diff(bms, loops, track, old, new):
'--benchmark_list_tests']).splitlines():
stripped_line = line.strip().replace("/", "_").replace(
"<", "_").replace(">", "_").replace(", ", "_")
js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
(bm, stripped_line, new, loop),
badjson_files, nonexistant_files)
js_new_opt = _read_json('%s.%s.opt.%s.%d.json' %
(bm, stripped_line, new, loop),
badjson_files, nonexistant_files)
js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
(bm, stripped_line, old, loop),
badjson_files, nonexistant_files)
js_old_opt = _read_json('%s.%s.opt.%s.%d.json' %
(bm, stripped_line, old, loop),
badjson_files, nonexistant_files)
if counters:
js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
(bm, stripped_line, new, loop),
badjson_files, nonexistant_files)
js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
(bm, stripped_line, old, loop),
badjson_files, nonexistant_files)
else:
js_new_ctr = None
js_old_ctr = None
if js_new_ctr:
for row in bm_json.expand_json(js_new_ctr, js_new_opt):
@ -187,12 +201,12 @@ def diff(bms, loops, track, old, new):
rows.append([name] + benchmarks[name].row(fields))
note = None
if len(badjson_files):
note = 'Corrupt JSON data (indicates timeout or crash) = %s' % str(badjson_files)
note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(badjson_files)
if len(nonexistant_files):
if note:
note += '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files)
note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
else:
note = '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files)
note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files)
if rows:
return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note
else:
@ -202,5 +216,5 @@ def diff(bms, loops, track, old, new):
if __name__ == '__main__':
args = _args()
diff, note = diff(args.benchmarks, args.loops, args.track, args.old,
args.new)
args.new, args.counters)
print('%s\n%s' % (note, diff if diff else "No performance differences"))

13
tools/profiling/microbenchmarks/bm_diff/bm_main.py
Unescape View File

@ -80,6 +80,9 @@ def _args():
type=int,
default=multiprocessing.cpu_count(),
help='Number of CPUs to use')
argp.add_argument('--counters', dest='counters', action='store_true')
argp.add_argument('--no-counters', dest='counters', action='store_false')
argp.set_defaults(counters=True)
args = argp.parse_args()
assert args.diff_base or args.old, "One of diff_base or old must be set!"
if args.loops < 3:
@ -103,7 +106,7 @@ def eintr_be_gone(fn):
def main(args):
bm_build.build('new', args.benchmarks, args.jobs)
bm_build.build('new', args.benchmarks, args.jobs, args.counters)
old = args.old
if args.diff_base:
@ -112,16 +115,16 @@ def main(args):
['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
bm_build.build('old', args.benchmarks, args.jobs)
bm_build.build(old, args.benchmarks, args.jobs, args.counters)
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions)
bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions)
bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)
bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)
diff, note = bm_diff.diff(args.benchmarks, args.loops, args.track, old,
'new')
'new', args.counters)
if diff:
text = 'Performance differences noted:\n' + diff
else:

17
tools/profiling/microbenchmarks/bm_diff/bm_run.py
Unescape View File

@ -67,6 +67,12 @@ def _args():
default=20,
help='Number of times to loops the benchmarks. More loops cuts down on noise'
)
argp.add_argument(
'-c',
'--counters',
type=bool,
default=True,
help='Whether or not to run and diff a counters build')
args = argp.parse_args()
assert args.name
if args.loops < 3:
@ -93,21 +99,22 @@ def _collect_bm_data(bm, cfg, name, reps, idx, loops):
shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1,
loops),
verbose_success=True,
timeout_seconds=60 * 2))
timeout_seconds=60 * 60)) # one hour
return jobs_list
def run(name, benchmarks, jobs, loops, reps):
def run(name, benchmarks, jobs, loops, reps, counters):
jobs_list = []
for loop in range(0, loops):
for bm in benchmarks:
jobs_list += _collect_bm_data(bm, 'opt', name, reps, loop, loops)
jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop,
loops)
if counters:
jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop,
loops)
random.shuffle(jobs_list, random.SystemRandom().random)
jobset.run(jobs_list, maxjobs=jobs)
if __name__ == '__main__':
args = _args()
run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions)
run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters)

Write Preview
Loading…
Cancel
Save