grpc/tools/profiling/microbenchmarks/bm_diff/bm_run.py

#!/usr/bin/env python2.7
#
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Python utility to run opt and counters benchmarks and save json output """

import bm_constants

import argparse
import subprocess
import multiprocessing
import random
import itertools
import sys
import os

sys.path.append(
    os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
                 'python_utils'))
import jobset


def _args():
    argp = argparse.ArgumentParser(description='Runs microbenchmarks')
    argp.add_argument('-b',
                      '--benchmarks',
                      nargs='+',
                      choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
                      default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
                      help='Benchmarks to run')
    argp.add_argument('-j',
                      '--jobs',
                      type=int,
                      default=multiprocessing.cpu_count(),
                      help='Number of CPUs to use')
    argp.add_argument(
        '-n',
        '--name',
        type=str,
        help=
        'Unique name of the build to run. Needs to match the handle passed to bm_build.py'
    )
    argp.add_argument('-r',
                      '--regex',
                      type=str,
                      default="",
                      help='Regex to filter benchmarks run')
    argp.add_argument(
        '-l',
        '--loops',
        type=int,
        default=20,
        help=
        'Number of times to loops the benchmarks. More loops cuts down on noise'
    )
    argp.add_argument('--counters', dest='counters', action='store_true')
    argp.add_argument('--no-counters', dest='counters', action='store_false')
    argp.set_defaults(counters=True)
    args = argp.parse_args()
    assert args.name
    if args.loops < 3:
        print "WARNING: This run will likely be noisy. Increase loops to at least 3."
    return args


def _collect_bm_data(bm, cfg, name, regex, idx, loops):
    jobs_list = []
    for line in subprocess.check_output([
            'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_list_tests',
            '--benchmark_filter=%s' % regex
    ]).splitlines():
        stripped_line = line.strip().replace("/",
                                             "_").replace("<", "_").replace(
                                                 ">", "_").replace(", ", "_")
        cmd = [
            'bm_diff_%s/%s/%s' % (name, cfg, bm),
            '--benchmark_filter=^%s$' % line,
            '--benchmark_out=%s.%s.%s.%s.%d.json' %
            (bm, stripped_line, cfg, name, idx),
            '--benchmark_out_format=json',
        ]
        jobs_list.append(
            jobset.JobSpec(cmd,
                           shortname='%s %s %s %s %d/%d' %
                           (bm, line, cfg, name, idx + 1, loops),
                           verbose_success=True,
                           cpu_cost=2,
                           timeout_seconds=60 * 60))  # one hour
    return jobs_list


def create_jobs(name, benchmarks, loops, regex, counters):
    jobs_list = []
    for loop in range(0, loops):
        for bm in benchmarks:
            jobs_list += _collect_bm_data(bm, 'opt', name, regex, loop, loops)
            if counters:
                jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop,
                                              loops)
    random.shuffle(jobs_list, random.SystemRandom().random)
    return jobs_list


if __name__ == '__main__':
    args = _args()
    jobs_list = create_jobs(args.name, args.benchmarks, args.loops, args.regex,
                            args.counters)
    jobset.run(jobs_list, maxjobs=args.jobs)