diff --git a/tools/jenkins/run_performance_flamegraphs.sh b/tools/jenkins/run_performance_flamegraphs.sh new file mode 100755 index 00000000000..22081e22487 --- /dev/null +++ b/tools/jenkins/run_performance_flamegraphs.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and runs full performance test suite. +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../.. + +# scalability with 32cores c++ benchmarks +tools/run_tests/run_performance_tests.py \ + -l c++ \ + --category scalable \ + --remote_worker_host grpc-performance-server-32core grpc-performance-client-32core grpc-performance-client2-32core \ + --perf_args "record -F 97 --call-graph dwarf" \ + --flame_graph_reports cpp_flamegraphs \ + || EXIT_CODE=1 + +# scalability with 32cores go benchmarks +tools/run_tests/run_performance_tests.py \ + -l go \ + --category scalable \ + --remote_worker_host grpc-performance-server-32core grpc-performance-client-32core grpc-performance-client2-32core \ + --perf_args "record -F 97 -g" \ + --flame_graph_reports go_flamegraphs \ + || EXIT_CODE=1 + +exit $EXIT_CODE + diff --git a/tools/jenkins/run_sweep_performance.sh b/tools/jenkins/run_sweep_performance.sh index 3b22e25a451..12054dc3dc8 100755 --- a/tools/jenkins/run_sweep_performance.sh +++ b/tools/jenkins/run_sweep_performance.sh @@ -43,6 +43,7 @@ tools/run_tests/run_performance_tests.py \ --category sweep \ --bq_result_table performance_test.performance_experiment_32core \ --remote_worker_host ${SERVER_HOST} ${CLIENT_HOST1} ${CLIENT_HOST2} \ + --perf_args "record -F 97 --call-graph dwarf" \ || EXIT_CODE=1 exit $EXIT_CODE diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 69ccff85cf5..06aa42756ef 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -58,8 +58,6 @@ os.chdir(_ROOT) _REMOTE_HOST_USERNAME = 'jenkins' -_PERF_REPORT_OUTPUT_DIR = 'perf_reports' - class QpsWorkerJob: """Encapsulates a qps worker server job.""" @@ -300,11 +298,11 @@ def perf_report_processor_job(worker_host, perf_base_name, output_filename): user_at_host = "%s@%s" % (_REMOTE_HOST_USERNAME, worker_host) cmd = "USER_AT_HOST=%s OUTPUT_FILENAME=%s OUTPUT_DIR=%s PERF_BASE_NAME=%s\ tools/run_tests/performance/process_remote_perf_flamegraphs.sh" \ - % (user_at_host, output_filename, _PERF_REPORT_OUTPUT_DIR, perf_base_name) + % (user_at_host, output_filename, args.flame_graph_reports, perf_base_name) else: cmd = "OUTPUT_FILENAME=%s OUTPUT_DIR=%s PERF_BASE_NAME=%s\ tools/run_tests/performance/process_local_perf_flamegraphs.sh" \ - % (output_filename, _PERF_REPORT_OUTPUT_DIR, perf_base_name) + % (output_filename, args.flame_graph_reports, perf_base_name) return jobset.JobSpec(cmdline=cmd, timeout_seconds=3*60, @@ -469,7 +467,7 @@ argp.add_argument('--perf_args', 'with the arguments to perf specified here. ' '".svg" flame graph profiles will be ' 'created for each Qps Worker on each scenario. ' - 'Files will output to "/perf_reports" ' + 'Files will output to "/" ' 'directory. Output files from running the worker ' 'under perf are saved in the repo root where its ran. ' 'Note that the perf "-g" flag is necessary for ' @@ -489,6 +487,8 @@ argp.add_argument('--skip_generate_flamegraphs', help=('Turn flame graph generation off. ' 'May be useful if "perf_args" arguments do not make sense for ' 'generating flamegraphs (e.g., "--perf_args=stat ...")')) +argp.add_argument('-f', '--flame_graph_reports', default='perf_reports', type=str, + help='Name of directory to output flame graph profiles to, if any are created.') args = argp.parse_args() @@ -522,8 +522,9 @@ if not args.dry_run: perf_cmd = None if args.perf_args: + print('Running workers under perf profiler') # Expect /usr/bin/perf to be installed here, as is usual - perf_cmd = ['/usr/bin/perf'] + perf_cmd = ['/usr/bin/perf'] perf_cmd.extend(re.split('\s+', args.perf_args)) qpsworker_jobs = create_qpsworkers(languages, args.remote_worker_host, perf_cmd=perf_cmd) @@ -582,7 +583,7 @@ for scenario in scenarios: # 'profile_output_files' will only have names for scenarios that passed if perf_cmd and not args.skip_generate_flamegraphs: # write the index fil to the output dir, with all profiles from all scenarios/workers - report_utils.render_perf_profiling_results('%s/index.html' % _PERF_REPORT_OUTPUT_DIR, profile_output_files) + report_utils.render_perf_profiling_results('%s/index.html' % args.flame_graph_reports, profile_output_files) if total_scenario_failures > 0 or qps_workers_killed > 0: print('%s scenarios failed and %s qps worker jobs killed' % (total_scenario_failures, qps_workers_killed))