|
|
|
@ -281,18 +281,18 @@ def create_qpsworkers(languages, worker_hosts, perf_cmd=None): |
|
|
|
for worker_idx, worker in enumerate(workers)] |
|
|
|
for worker_idx, worker in enumerate(workers)] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def perf_report_processor_job(worker_host, perf_base_name, output_filename): |
|
|
|
def perf_report_processor_job(worker_host, perf_base_name, output_filename, flame_graph_reports): |
|
|
|
print('Creating perf report collection job for %s' % worker_host) |
|
|
|
print('Creating perf report collection job for %s' % worker_host) |
|
|
|
cmd = '' |
|
|
|
cmd = '' |
|
|
|
if worker_host != 'localhost': |
|
|
|
if worker_host != 'localhost': |
|
|
|
user_at_host = "%s@%s" % (_REMOTE_HOST_USERNAME, worker_host) |
|
|
|
user_at_host = "%s@%s" % (_REMOTE_HOST_USERNAME, worker_host) |
|
|
|
cmd = "USER_AT_HOST=%s OUTPUT_FILENAME=%s OUTPUT_DIR=%s PERF_BASE_NAME=%s\ |
|
|
|
cmd = "USER_AT_HOST=%s OUTPUT_FILENAME=%s OUTPUT_DIR=%s PERF_BASE_NAME=%s\ |
|
|
|
tools/run_tests/performance/process_remote_perf_flamegraphs.sh" \ |
|
|
|
tools/run_tests/performance/process_remote_perf_flamegraphs.sh" \ |
|
|
|
% (user_at_host, output_filename, args.flame_graph_reports, perf_base_name) |
|
|
|
% (user_at_host, output_filename, flame_graph_reports, perf_base_name) |
|
|
|
else: |
|
|
|
else: |
|
|
|
cmd = "OUTPUT_FILENAME=%s OUTPUT_DIR=%s PERF_BASE_NAME=%s\ |
|
|
|
cmd = "OUTPUT_FILENAME=%s OUTPUT_DIR=%s PERF_BASE_NAME=%s\ |
|
|
|
tools/run_tests/performance/process_local_perf_flamegraphs.sh" \ |
|
|
|
tools/run_tests/performance/process_local_perf_flamegraphs.sh" \ |
|
|
|
% (output_filename, args.flame_graph_reports, perf_base_name) |
|
|
|
% (output_filename, flame_graph_reports, perf_base_name) |
|
|
|
|
|
|
|
|
|
|
|
return jobset.JobSpec(cmdline=cmd, |
|
|
|
return jobset.JobSpec(cmdline=cmd, |
|
|
|
timeout_seconds=3*60, |
|
|
|
timeout_seconds=3*60, |
|
|
|
@ -332,7 +332,7 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', |
|
|
|
|
|
|
|
|
|
|
|
for language in languages: |
|
|
|
for language in languages: |
|
|
|
for scenario_json in language.scenarios(): |
|
|
|
for scenario_json in language.scenarios(): |
|
|
|
if re.search(args.regex, scenario_json['name']): |
|
|
|
if re.search(regex, scenario_json['name']): |
|
|
|
categories = scenario_json.get('CATEGORIES', ['scalable', 'smoketest']) |
|
|
|
categories = scenario_json.get('CATEGORIES', ['scalable', 'smoketest']) |
|
|
|
if category in categories or category == 'all': |
|
|
|
if category in categories or category == 'all': |
|
|
|
workers = workers_by_lang[str(language)][:] |
|
|
|
workers = workers_by_lang[str(language)][:] |
|
|
|
@ -376,7 +376,7 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', |
|
|
|
return scenarios |
|
|
|
return scenarios |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def finish_qps_workers(jobs): |
|
|
|
def finish_qps_workers(jobs, qpsworker_jobs): |
|
|
|
"""Waits for given jobs to finish and eventually kills them.""" |
|
|
|
"""Waits for given jobs to finish and eventually kills them.""" |
|
|
|
retries = 0 |
|
|
|
retries = 0 |
|
|
|
num_killed = 0 |
|
|
|
num_killed = 0 |
|
|
|
@ -402,7 +402,7 @@ profile_output_files = [] |
|
|
|
# perf reports directory. |
|
|
|
# perf reports directory. |
|
|
|
# Alos, the perf profiles need to be fetched and processed after each scenario |
|
|
|
# Alos, the perf profiles need to be fetched and processed after each scenario |
|
|
|
# in order to avoid clobbering the output files. |
|
|
|
# in order to avoid clobbering the output files. |
|
|
|
def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name): |
|
|
|
def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name, flame_graph_reports): |
|
|
|
perf_report_jobs = [] |
|
|
|
perf_report_jobs = [] |
|
|
|
global profile_output_files |
|
|
|
global profile_output_files |
|
|
|
for host_and_port in hosts_and_base_names: |
|
|
|
for host_and_port in hosts_and_base_names: |
|
|
|
@ -411,51 +411,51 @@ def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name): |
|
|
|
# from the base filename, create .svg output filename |
|
|
|
# from the base filename, create .svg output filename |
|
|
|
host = host_and_port.split(':')[0] |
|
|
|
host = host_and_port.split(':')[0] |
|
|
|
profile_output_files.append('%s.svg' % output_filename) |
|
|
|
profile_output_files.append('%s.svg' % output_filename) |
|
|
|
perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename)) |
|
|
|
perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename, flame_graph_reports)) |
|
|
|
|
|
|
|
|
|
|
|
jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) |
|
|
|
jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) |
|
|
|
failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) |
|
|
|
failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1, clear_alarms=False) |
|
|
|
jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) |
|
|
|
jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) |
|
|
|
return failures |
|
|
|
return failures |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main(): |
|
|
|
argp = argparse.ArgumentParser(description='Run performance tests.') |
|
|
|
argp = argparse.ArgumentParser(description='Run performance tests.') |
|
|
|
argp.add_argument('-l', '--language', |
|
|
|
argp.add_argument('-l', '--language', |
|
|
|
choices=['all'] + sorted(scenario_config.LANGUAGES.keys()), |
|
|
|
choices=['all'] + sorted(scenario_config.LANGUAGES.keys()), |
|
|
|
nargs='+', |
|
|
|
nargs='+', |
|
|
|
required=True, |
|
|
|
required=True, |
|
|
|
help='Languages to benchmark.') |
|
|
|
help='Languages to benchmark.') |
|
|
|
argp.add_argument('--remote_driver_host', |
|
|
|
argp.add_argument('--remote_driver_host', |
|
|
|
default=None, |
|
|
|
default=None, |
|
|
|
help='Run QPS driver on given host. By default, QPS driver is run locally.') |
|
|
|
help='Run QPS driver on given host. By default, QPS driver is run locally.') |
|
|
|
argp.add_argument('--remote_worker_host', |
|
|
|
argp.add_argument('--remote_worker_host', |
|
|
|
nargs='+', |
|
|
|
nargs='+', |
|
|
|
default=[], |
|
|
|
default=[], |
|
|
|
help='Worker hosts where to start QPS workers.') |
|
|
|
help='Worker hosts where to start QPS workers.') |
|
|
|
argp.add_argument('--dry_run', |
|
|
|
argp.add_argument('--dry_run', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Just list scenarios to be run, but don\'t run them.') |
|
|
|
help='Just list scenarios to be run, but don\'t run them.') |
|
|
|
argp.add_argument('-r', '--regex', default='.*', type=str, |
|
|
|
argp.add_argument('-r', '--regex', default='.*', type=str, |
|
|
|
help='Regex to select scenarios to run.') |
|
|
|
help='Regex to select scenarios to run.') |
|
|
|
argp.add_argument('--bq_result_table', default=None, type=str, |
|
|
|
argp.add_argument('--bq_result_table', default=None, type=str, |
|
|
|
help='Bigquery "dataset.table" to upload results to.') |
|
|
|
help='Bigquery "dataset.table" to upload results to.') |
|
|
|
argp.add_argument('--category', |
|
|
|
argp.add_argument('--category', |
|
|
|
choices=['smoketest','all','scalable','sweep'], |
|
|
|
choices=['smoketest','all','scalable','sweep'], |
|
|
|
default='all', |
|
|
|
default='all', |
|
|
|
help='Select a category of tests to run.') |
|
|
|
help='Select a category of tests to run.') |
|
|
|
argp.add_argument('--netperf', |
|
|
|
argp.add_argument('--netperf', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help='Run netperf benchmark as one of the scenarios.') |
|
|
|
help='Run netperf benchmark as one of the scenarios.') |
|
|
|
argp.add_argument('--server_cpu_load', |
|
|
|
argp.add_argument('--server_cpu_load', |
|
|
|
default=0, type=int, |
|
|
|
default=0, type=int, |
|
|
|
help='Select a targeted server cpu load to run. 0 means ignore this flag') |
|
|
|
help='Select a targeted server cpu load to run. 0 means ignore this flag') |
|
|
|
argp.add_argument('-x', '--xml_report', default='report.xml', type=str, |
|
|
|
argp.add_argument('-x', '--xml_report', default='report.xml', type=str, |
|
|
|
help='Name of XML report file to generate.') |
|
|
|
help='Name of XML report file to generate.') |
|
|
|
argp.add_argument('--perf_args', |
|
|
|
argp.add_argument('--perf_args', |
|
|
|
help=('Example usage: "--perf_args=record -F 99 -g". ' |
|
|
|
help=('Example usage: "--perf_args=record -F 99 -g". ' |
|
|
|
'Wrap QPS workers in a perf command ' |
|
|
|
'Wrap QPS workers in a perf command ' |
|
|
|
'with the arguments to perf specified here. ' |
|
|
|
'with the arguments to perf specified here. ' |
|
|
|
@ -474,60 +474,60 @@ argp.add_argument('--perf_args', |
|
|
|
'not be used at all. ' |
|
|
|
'not be used at all. ' |
|
|
|
'See http://www.brendangregg.com/perf.html ' |
|
|
|
'See http://www.brendangregg.com/perf.html ' |
|
|
|
'for more general perf examples.')) |
|
|
|
'for more general perf examples.')) |
|
|
|
argp.add_argument('--skip_generate_flamegraphs', |
|
|
|
argp.add_argument('--skip_generate_flamegraphs', |
|
|
|
default=False, |
|
|
|
default=False, |
|
|
|
action='store_const', |
|
|
|
action='store_const', |
|
|
|
const=True, |
|
|
|
const=True, |
|
|
|
help=('Turn flame graph generation off. ' |
|
|
|
help=('Turn flame graph generation off. ' |
|
|
|
'May be useful if "perf_args" arguments do not make sense for ' |
|
|
|
'May be useful if "perf_args" arguments do not make sense for ' |
|
|
|
'generating flamegraphs (e.g., "--perf_args=stat ...")')) |
|
|
|
'generating flamegraphs (e.g., "--perf_args=stat ...")')) |
|
|
|
argp.add_argument('-f', '--flame_graph_reports', default='perf_reports', type=str, |
|
|
|
argp.add_argument('-f', '--flame_graph_reports', default='perf_reports', type=str, |
|
|
|
help='Name of directory to output flame graph profiles to, if any are created.') |
|
|
|
help='Name of directory to output flame graph profiles to, if any are created.') |
|
|
|
|
|
|
|
|
|
|
|
args = argp.parse_args() |
|
|
|
args = argp.parse_args() |
|
|
|
|
|
|
|
|
|
|
|
languages = set(scenario_config.LANGUAGES[l] |
|
|
|
languages = set(scenario_config.LANGUAGES[l] |
|
|
|
for l in itertools.chain.from_iterable( |
|
|
|
for l in itertools.chain.from_iterable( |
|
|
|
six.iterkeys(scenario_config.LANGUAGES) if x == 'all' |
|
|
|
six.iterkeys(scenario_config.LANGUAGES) if x == 'all' |
|
|
|
else [x] for x in args.language)) |
|
|
|
else [x] for x in args.language)) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Put together set of remote hosts where to run and build |
|
|
|
# Put together set of remote hosts where to run and build |
|
|
|
remote_hosts = set() |
|
|
|
remote_hosts = set() |
|
|
|
if args.remote_worker_host: |
|
|
|
if args.remote_worker_host: |
|
|
|
for host in args.remote_worker_host: |
|
|
|
for host in args.remote_worker_host: |
|
|
|
remote_hosts.add(host) |
|
|
|
remote_hosts.add(host) |
|
|
|
if args.remote_driver_host: |
|
|
|
if args.remote_driver_host: |
|
|
|
remote_hosts.add(args.remote_driver_host) |
|
|
|
remote_hosts.add(args.remote_driver_host) |
|
|
|
|
|
|
|
|
|
|
|
if not args.dry_run: |
|
|
|
if not args.dry_run: |
|
|
|
if remote_hosts: |
|
|
|
if remote_hosts: |
|
|
|
archive_repo(languages=[str(l) for l in languages]) |
|
|
|
archive_repo(languages=[str(l) for l in languages]) |
|
|
|
prepare_remote_hosts(remote_hosts, prepare_local=True) |
|
|
|
prepare_remote_hosts(remote_hosts, prepare_local=True) |
|
|
|
else: |
|
|
|
else: |
|
|
|
prepare_remote_hosts([], prepare_local=True) |
|
|
|
prepare_remote_hosts([], prepare_local=True) |
|
|
|
|
|
|
|
|
|
|
|
build_local = False |
|
|
|
build_local = False |
|
|
|
if not args.remote_driver_host: |
|
|
|
if not args.remote_driver_host: |
|
|
|
build_local = True |
|
|
|
build_local = True |
|
|
|
if not args.dry_run: |
|
|
|
if not args.dry_run: |
|
|
|
build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local) |
|
|
|
build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local) |
|
|
|
|
|
|
|
|
|
|
|
perf_cmd = None |
|
|
|
perf_cmd = None |
|
|
|
if args.perf_args: |
|
|
|
if args.perf_args: |
|
|
|
print('Running workers under perf profiler') |
|
|
|
print('Running workers under perf profiler') |
|
|
|
# Expect /usr/bin/perf to be installed here, as is usual |
|
|
|
# Expect /usr/bin/perf to be installed here, as is usual |
|
|
|
perf_cmd = ['/usr/bin/perf'] |
|
|
|
perf_cmd = ['/usr/bin/perf'] |
|
|
|
perf_cmd.extend(re.split('\s+', args.perf_args)) |
|
|
|
perf_cmd.extend(re.split('\s+', args.perf_args)) |
|
|
|
|
|
|
|
|
|
|
|
qpsworker_jobs = create_qpsworkers(languages, args.remote_worker_host, perf_cmd=perf_cmd) |
|
|
|
qpsworker_jobs = create_qpsworkers(languages, args.remote_worker_host, perf_cmd=perf_cmd) |
|
|
|
|
|
|
|
|
|
|
|
# get list of worker addresses for each language. |
|
|
|
# get list of worker addresses for each language. |
|
|
|
workers_by_lang = dict([(str(language), []) for language in languages]) |
|
|
|
workers_by_lang = dict([(str(language), []) for language in languages]) |
|
|
|
for job in qpsworker_jobs: |
|
|
|
for job in qpsworker_jobs: |
|
|
|
workers_by_lang[str(job.language)].append(job) |
|
|
|
workers_by_lang[str(job.language)].append(job) |
|
|
|
|
|
|
|
|
|
|
|
scenarios = create_scenarios(languages, |
|
|
|
scenarios = create_scenarios(languages, |
|
|
|
workers_by_lang=workers_by_lang, |
|
|
|
workers_by_lang=workers_by_lang, |
|
|
|
remote_host=args.remote_driver_host, |
|
|
|
remote_host=args.remote_driver_host, |
|
|
|
regex=args.regex, |
|
|
|
regex=args.regex, |
|
|
|
@ -537,15 +537,15 @@ scenarios = create_scenarios(languages, |
|
|
|
netperf_hosts=args.remote_worker_host, |
|
|
|
netperf_hosts=args.remote_worker_host, |
|
|
|
server_cpu_load=args.server_cpu_load) |
|
|
|
server_cpu_load=args.server_cpu_load) |
|
|
|
|
|
|
|
|
|
|
|
if not scenarios: |
|
|
|
if not scenarios: |
|
|
|
raise Exception('No scenarios to run') |
|
|
|
raise Exception('No scenarios to run') |
|
|
|
|
|
|
|
|
|
|
|
total_scenario_failures = 0 |
|
|
|
total_scenario_failures = 0 |
|
|
|
qps_workers_killed = 0 |
|
|
|
qps_workers_killed = 0 |
|
|
|
merged_resultset = {} |
|
|
|
merged_resultset = {} |
|
|
|
perf_report_failures = 0 |
|
|
|
perf_report_failures = 0 |
|
|
|
|
|
|
|
|
|
|
|
for scenario in scenarios: |
|
|
|
for scenario in scenarios: |
|
|
|
if args.dry_run: |
|
|
|
if args.dry_run: |
|
|
|
print(scenario.name) |
|
|
|
print(scenario.name) |
|
|
|
else: |
|
|
|
else: |
|
|
|
@ -562,7 +562,7 @@ for scenario in scenarios: |
|
|
|
six.iteritems(resultset))) |
|
|
|
six.iteritems(resultset))) |
|
|
|
finally: |
|
|
|
finally: |
|
|
|
# Consider qps workers that need to be killed as failures |
|
|
|
# Consider qps workers that need to be killed as failures |
|
|
|
qps_workers_killed += finish_qps_workers(scenario.workers) |
|
|
|
qps_workers_killed += finish_qps_workers(scenario.workers, qpsworker_jobs) |
|
|
|
|
|
|
|
|
|
|
|
if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs: |
|
|
|
if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs: |
|
|
|
workers_and_base_names = {} |
|
|
|
workers_and_base_names = {} |
|
|
|
@ -570,22 +570,25 @@ for scenario in scenarios: |
|
|
|
if not worker.perf_file_base_name: |
|
|
|
if not worker.perf_file_base_name: |
|
|
|
raise Exception('using perf buf perf report filename is unspecified') |
|
|
|
raise Exception('using perf buf perf report filename is unspecified') |
|
|
|
workers_and_base_names[worker.host_and_port] = worker.perf_file_base_name |
|
|
|
workers_and_base_names[worker.host_and_port] = worker.perf_file_base_name |
|
|
|
perf_report_failures += run_collect_perf_profile_jobs(workers_and_base_names, scenario.name) |
|
|
|
perf_report_failures += run_collect_perf_profile_jobs(workers_and_base_names, scenario.name, args.flame_graph_reports) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Still write the index.html even if some scenarios failed. |
|
|
|
# Still write the index.html even if some scenarios failed. |
|
|
|
# 'profile_output_files' will only have names for scenarios that passed |
|
|
|
# 'profile_output_files' will only have names for scenarios that passed |
|
|
|
if perf_cmd and not args.skip_generate_flamegraphs: |
|
|
|
if perf_cmd and not args.skip_generate_flamegraphs: |
|
|
|
# write the index fil to the output dir, with all profiles from all scenarios/workers |
|
|
|
# write the index fil to the output dir, with all profiles from all scenarios/workers |
|
|
|
report_utils.render_perf_profiling_results('%s/index.html' % args.flame_graph_reports, profile_output_files) |
|
|
|
report_utils.render_perf_profiling_results('%s/index.html' % args.flame_graph_reports, profile_output_files) |
|
|
|
|
|
|
|
|
|
|
|
report_utils.render_junit_xml_report(merged_resultset, args.xml_report, |
|
|
|
report_utils.render_junit_xml_report(merged_resultset, args.xml_report, |
|
|
|
suite_name='benchmarks') |
|
|
|
suite_name='benchmarks') |
|
|
|
|
|
|
|
|
|
|
|
if total_scenario_failures > 0 or qps_workers_killed > 0: |
|
|
|
if total_scenario_failures > 0 or qps_workers_killed > 0: |
|
|
|
print('%s scenarios failed and %s qps worker jobs killed' % (total_scenario_failures, qps_workers_killed)) |
|
|
|
print('%s scenarios failed and %s qps worker jobs killed' % (total_scenario_failures, qps_workers_killed)) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
if perf_report_failures > 0: |
|
|
|
if perf_report_failures > 0: |
|
|
|
print('%s perf profile collection jobs failed' % perf_report_failures) |
|
|
|
print('%s perf profile collection jobs failed' % perf_report_failures) |
|
|
|
sys.exit(1) |
|
|
|
sys.exit(1) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__": |
|
|
|
|
|
|
|
main() |
|
|
|
|
Craig Tiller
7 years ago