Merge pull request #10079 from matt-kwong/change-to-microbenchmarking

Change performance testing to use benchmark diff tool
reviewable/pr10259/r5^2
Matt Kwong 8 years ago committed by GitHub
commit 9d67ee7222
  1. 45
      tools/jenkins/comment_on_pr.sh
  2. 5
      tools/jenkins/run_performance.sh
  3. 36
      tools/run_tests/run_microbenchmark.py

@ -0,0 +1,45 @@
#!/usr/bin/env bash
# Copyright 2017, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# This script is invoked by Jenkins to comment $1 on pull requests
# when triggered by a build
set -e
if [ -z "$1" ] || [ -z $JENKINS_OAUTH_TOKEN ] || [ -z $ghprbPullId ]; then
echo "Insufficient arguments or environment variables provided."
exit 1
fi
# Format the comment message to JSON
COMMENT_MESSAGE="{\"body\":\"$1\"}"
curl -k -H "Authorization: token $JENKINS_OAUTH_TOKEN" -H "Content-Type: application/json" \
-d "$COMMENT_MESSAGE" https://api.github.com/repos/grpc/grpc/issues/$ghprbPullId/comments

@ -31,7 +31,12 @@
# This script is invoked by Jenkins and runs performance smoke test.
set -ex
# List of benchmarks that provide good signal for analyzing performance changes in pull requests
BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
# Enter the gRPC repo root
cd $(dirname $0)/../..
tools/run_tests/run_performance_tests.py -l c++ node ruby csharp python --netperf --category smoketest
# todo(mattkwong): Change performance test to use microbenchmarking
# tools/run_tests/run_microbenchmark.py -c summary --diff_perf origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN

@ -38,6 +38,19 @@ import argparse
import python_utils.jobset as jobset
import python_utils.start_port_server as start_port_server
_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong',
'bm_fullstack_streaming_ping_pong',
'bm_fullstack_streaming_pump',
'bm_closure',
'bm_cq',
'bm_call_create',
'bm_error',
'bm_chttp2_hpack',
'bm_chttp2_transport',
'bm_pollset',
'bm_metadata',
'bm_fullstack_trickle']
flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')
os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
@ -201,19 +214,8 @@ argp.add_argument('-c', '--collect',
default=sorted(collectors.keys()),
help='Which collectors should be run against each benchmark')
argp.add_argument('-b', '--benchmarks',
default=['bm_fullstack_unary_ping_pong',
'bm_fullstack_streaming_ping_pong',
'bm_fullstack_streaming_pump',
'bm_closure',
'bm_cq',
'bm_call_create',
'bm_error',
'bm_chttp2_hpack',
'bm_chttp2_transport',
'bm_pollset',
'bm_metadata',
'bm_fullstack_trickle',
],
choices=_AVAILABLE_BENCHMARK_TESTS,
default=_AVAILABLE_BENCHMARK_TESTS,
nargs='+',
type=str,
help='Which microbenchmarks should be run')
@ -237,6 +239,7 @@ try:
for bm_name in args.benchmarks:
collectors[collect](bm_name, args)
if args.diff_perf:
git_comment = 'Performance differences between this PR and %s\\n' % args.diff_perf
if 'summary' not in args.collect:
for bm_name in args.benchmarks:
run_summary(bm_name, 'opt', bm_name)
@ -264,7 +267,14 @@ try:
if diff:
heading('Performance diff: %s' % bm_name)
text(diff)
git_comment += '```\\nPerformance diff: %s\\n%s\\n```\\n' % (bm_name, diff.replace('\n', '\\n'))
finally:
if args.diff_perf:
subprocess.call(['tools/jenkins/comment_on_pr.sh "%s"' % git_comment.replace('`', '\`')],
stdout=subprocess.PIPE,
shell=True)
if not os.path.exists('reports'):
os.makedirs('reports')
index_html += "</body>\n</html>\n"
with open('reports/index.html', 'w') as f:
f.write(index_html)

Loading…
Cancel
Save