From 63aa9c8446e194df733c1845211c0862d5f00ba7 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Wed, 14 Jun 2017 16:58:09 -0700 Subject: [PATCH 1/5] Add QPS Diff --- test/cpp/qps/qps_diff.py | 164 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100755 test/cpp/qps/qps_diff.py diff --git a/test/cpp/qps/qps_diff.py b/test/cpp/qps/qps_diff.py new file mode 100755 index 00000000000..9db5f731942 --- /dev/null +++ b/test/cpp/qps/qps_diff.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python2.7 +# +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Computes the diff between two qps runs and outputs significant results """ + +import shutil +import subprocess +import os +import multiprocessing +import json +import sys +import tabulate +import argparse + +sys.path.append( + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', '..', 'tools', 'profiling', 'microbenchmarks', 'bm_diff')) +import bm_speedup + +sys.path.append( + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', '..', 'tools', 'run_tests', 'python_utils')) +import comment_on_pr + +_SCENARIOS = { + 'large-message-throughput': '{"scenarios":[{"name":"large-message-throughput", "spawn_local_worker_count": -2, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}', + 'multi-channel-64-KiB': '{"scenarios":[{"name":"multi-channel-64-KiB", "spawn_local_worker_count": -3, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 31, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 2, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 32, "async_client_threads": 31, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}' +} + +def _args(): + argp = argparse.ArgumentParser( + description='Perform diff on QPS Driver') + argp.add_argument( + '-d', + '--diff_base', + type=str, + help='Commit or branch to compare the current one to') + argp.add_argument( + '-l', + '--loops', + type=int, + default=6, + help='Number of times to loops the benchmarks. More loops cuts down on noise' + ) + argp.add_argument( + '-j', + '--jobs', + type=int, + default=multiprocessing.cpu_count(), + help='Number of CPUs to use') + args = argp.parse_args() + assert args.diff_base, "diff_base must be set" + return args + +def _make_cmd(jobs): + return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker',] + +def build(name, jobs): + shutil.rmtree('qps_diff_%s' % name, ignore_errors=True) + subprocess.check_call(['git', 'submodule', 'update']) + try: + subprocess.check_call(_make_cmd(jobs)) + except subprocess.CalledProcessError, e: + subprocess.check_call(['make', 'clean']) + subprocess.check_call(_make_cmd(jobs)) + os.rename('bins', 'qps_diff_%s' % name) + +def _run_cmd(name, scenario, fname): + return ['qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario, '--json_file_out', fname] + +def run(name, scenarios, loops): + for sn in scenarios: + for i in range(0, loops): + fname = "%s.%s.%d.json" % (sn, name, i) + subprocess.check_call(_run_cmd(name, scenarios[sn], fname)) + +def _load_qps(fname): + try: + with open(fname) as f: + return json.loads(f.read())['qps'] + except IOError, e: + print("IOError occurred reading file: %s" % fname) + return None + except ValueError, e: + print("ValueError occurred reading file: %s" % fname) + return None + +def _median(ary): + assert (len(ary)) + ary = sorted(ary) + n = len(ary) + if n % 2 == 0: + return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0 + else: + return ary[n / 2] + +def diff(scenarios, loops, old, new): + old_data = {} + new_data = {} + + # collect data + for sn in scenarios: + old_data[sn] = [] + new_data[sn] = [] + for i in range(0, loops): + old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i))) + new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i))) + + # crunch data + headers = ['Benchmark', 'qps'] + rows = [] + for sn in scenarios: + mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn])) + print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old, old_data[sn], mdn_diff)) + s = bm_speedup.speedup(new_data[sn], old_data[sn]) + if abs(s) > 3 and mdn_diff > 0.5: + rows.append([sn, '%+d%%' % s]) + + if rows: + return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') + else: + return None + +def main(args): + + build('new', args.jobs) + + if args.diff_base: + where_am_i = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_base]) + try: + build('old', args.jobs) + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) + + run('new', _SCENARIOS, args.loops) + run('old', _SCENARIOS, args.loops) + + diff = diff(_SCENARIOS, args.loops, 'old', 'new') + + if diff: + text = '[qps] Performance differences noted:\n%s' % diff + else: + text = '[qps] No significant performance differences' + print('%s' % text) + comment_on_pr.comment_on_pr('```\n%s\n```' % text) + +if __name__ == '__main__': + args = _args() + main(args); From 390668999f9af7997e9e49995333a56516738da6 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Fri, 16 Jun 2017 15:12:52 -0700 Subject: [PATCH 2/5] relax QPS diff threshold --- test/cpp/qps/qps_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/qps_diff.py b/test/cpp/qps/qps_diff.py index 9db5f731942..a95f9b899ae 100755 --- a/test/cpp/qps/qps_diff.py +++ b/test/cpp/qps/qps_diff.py @@ -124,7 +124,7 @@ def diff(scenarios, loops, old, new): for sn in scenarios: mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn])) print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old, old_data[sn], mdn_diff)) - s = bm_speedup.speedup(new_data[sn], old_data[sn]) + s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5) if abs(s) > 3 and mdn_diff > 0.5: rows.append([sn, '%+d%%' % s]) From 61a75993f5fca316a3636f4c7a44ae5831a9bec9 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 19 Jun 2017 09:05:34 -0700 Subject: [PATCH 3/5] Enable jenkins testing --- test/cpp/qps/qps_diff.py | 8 ++++---- tools/jenkins/run_performance.sh | 7 ++----- tools/jenkins/run_performance_old.sh | 26 ++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 9 deletions(-) create mode 100755 tools/jenkins/run_performance_old.sh diff --git a/test/cpp/qps/qps_diff.py b/test/cpp/qps/qps_diff.py index a95f9b899ae..1dadd96adfb 100755 --- a/test/cpp/qps/qps_diff.py +++ b/test/cpp/qps/qps_diff.py @@ -51,7 +51,7 @@ def _args(): '-l', '--loops', type=int, - default=6, + default=4, help='Number of times to loops the benchmarks. More loops cuts down on noise' ) argp.add_argument( @@ -150,10 +150,10 @@ def main(args): run('new', _SCENARIOS, args.loops) run('old', _SCENARIOS, args.loops) - diff = diff(_SCENARIOS, args.loops, 'old', 'new') + diff_output = diff(_SCENARIOS, args.loops, 'old', 'new') - if diff: - text = '[qps] Performance differences noted:\n%s' % diff + if diff_output: + text = '[qps] Performance differences noted:\n%s' % diff_output else: text = '[qps] No significant performance differences' print('%s' % text) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 3ce05cc7f1e..3673fa6c405 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -13,14 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# This script is invoked by Jenkins and runs a diff on the microbenchmarks +# This script is invoked by Jenkins and runs a diff on the qps drivers set -ex -# List of benchmarks that provide good signal for analyzing performance changes in pull requests -BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" - # Enter the gRPC repo root cd $(dirname $0)/../.. tools/run_tests/start_port_server.py -tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN +test/cpp/qps/qps_diff.py -d origin/$ghprbTargetBranch diff --git a/tools/jenkins/run_performance_old.sh b/tools/jenkins/run_performance_old.sh new file mode 100755 index 00000000000..3ce05cc7f1e --- /dev/null +++ b/tools/jenkins/run_performance_old.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on the microbenchmarks +set -ex + +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + +# Enter the gRPC repo root +cd $(dirname $0)/../.. + +tools/run_tests/start_port_server.py +tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN From 06795cb442226dd9b6766bc3adcd63aa15c9b10c Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 19 Jun 2017 20:14:23 -0700 Subject: [PATCH 4/5] Address github comments --- tools/jenkins/run_performance.sh | 2 +- tools/profiling/microbenchmarks/bm_json.py | 2 +- {test/cpp => tools/profiling}/qps/qps_diff.py | 39 +++++++++++-------- tools/profiling/qps/qps_scenarios.py | 21 ++++++++++ 4 files changed, 45 insertions(+), 19 deletions(-) rename {test/cpp => tools/profiling}/qps/qps_diff.py (70%) create mode 100644 tools/profiling/qps/qps_scenarios.py diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 3673fa6c405..9529b0126fc 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -20,4 +20,4 @@ set -ex cd $(dirname $0)/../.. tools/run_tests/start_port_server.py -test/cpp/qps/qps_diff.py -d origin/$ghprbTargetBranch +tools/profiling/qps/qps_diff.py -d origin/$ghprbTargetBranch diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index 930287e0d69..f6082fe7b4a 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -167,7 +167,7 @@ def parse_name(name): return out def expand_json(js, js2 = None): - assert(js or js2) + if not js and not js2: raise StopIteration() if not js: js = js2 for bm in js['benchmarks']: if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): continue diff --git a/test/cpp/qps/qps_diff.py b/tools/profiling/qps/qps_diff.py similarity index 70% rename from test/cpp/qps/qps_diff.py rename to tools/profiling/qps/qps_diff.py index 1dadd96adfb..65c845caa1f 100755 --- a/test/cpp/qps/qps_diff.py +++ b/tools/profiling/qps/qps_diff.py @@ -15,29 +15,26 @@ # limitations under the License. """ Computes the diff between two qps runs and outputs significant results """ +import argparse +import json +import multiprocessing +import os +import qps_scenarios import shutil import subprocess -import os -import multiprocessing -import json import sys import tabulate -import argparse sys.path.append( os.path.join( - os.path.dirname(sys.argv[0]), '..', '..', '..', 'tools', 'profiling', 'microbenchmarks', 'bm_diff')) + os.path.dirname(sys.argv[0]), '..', 'microbenchmarks', 'bm_diff')) import bm_speedup sys.path.append( os.path.join( - os.path.dirname(sys.argv[0]), '..', '..', '..', 'tools', 'run_tests', 'python_utils')) + os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr -_SCENARIOS = { - 'large-message-throughput': '{"scenarios":[{"name":"large-message-throughput", "spawn_local_worker_count": -2, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}', - 'multi-channel-64-KiB': '{"scenarios":[{"name":"multi-channel-64-KiB", "spawn_local_worker_count": -3, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 31, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 2, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 32, "async_client_threads": 31, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}' -} def _args(): argp = argparse.ArgumentParser( @@ -52,7 +49,7 @@ def _args(): '--loops', type=int, default=4, - help='Number of times to loops the benchmarks. More loops cuts down on noise' + help='Number of loops for each benchmark. More loops cuts down on noise' ) argp.add_argument( '-j', @@ -64,8 +61,10 @@ def _args(): assert args.diff_base, "diff_base must be set" return args + def _make_cmd(jobs): - return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker',] + return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker'] + def build(name, jobs): shutil.rmtree('qps_diff_%s' % name, ignore_errors=True) @@ -77,15 +76,18 @@ def build(name, jobs): subprocess.check_call(_make_cmd(jobs)) os.rename('bins', 'qps_diff_%s' % name) + def _run_cmd(name, scenario, fname): return ['qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario, '--json_file_out', fname] + def run(name, scenarios, loops): for sn in scenarios: for i in range(0, loops): fname = "%s.%s.%d.json" % (sn, name, i) subprocess.check_call(_run_cmd(name, scenarios[sn], fname)) + def _load_qps(fname): try: with open(fname) as f: @@ -97,6 +99,7 @@ def _load_qps(fname): print("ValueError occurred reading file: %s" % fname) return None + def _median(ary): assert (len(ary)) ary = sorted(ary) @@ -106,6 +109,7 @@ def _median(ary): else: return ary[n / 2] + def diff(scenarios, loops, old, new): old_data = {} new_data = {} @@ -133,8 +137,8 @@ def diff(scenarios, loops, old, new): else: return None -def main(args): +def main(args): build('new', args.jobs) if args.diff_base: @@ -147,10 +151,10 @@ def main(args): subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) - run('new', _SCENARIOS, args.loops) - run('old', _SCENARIOS, args.loops) + run('new', qps_scenarios._SCENARIOS, args.loops) + run('old', qps_scenarios._SCENARIOS, args.loops) - diff_output = diff(_SCENARIOS, args.loops, 'old', 'new') + diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new') if diff_output: text = '[qps] Performance differences noted:\n%s' % diff_output @@ -159,6 +163,7 @@ def main(args): print('%s' % text) comment_on_pr.comment_on_pr('```\n%s\n```' % text) + if __name__ == '__main__': args = _args() - main(args); + main(args) diff --git a/tools/profiling/qps/qps_scenarios.py b/tools/profiling/qps/qps_scenarios.py new file mode 100644 index 00000000000..d473d7cd240 --- /dev/null +++ b/tools/profiling/qps/qps_scenarios.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python2.7 +# +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" QPS Scenarios to run """ + +_SCENARIOS = { + 'large-message-throughput': '{"scenarios":[{"name":"large-message-throughput", "spawn_local_worker_count": -2, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}', + 'multi-channel-64-KiB': '{"scenarios":[{"name":"multi-channel-64-KiB", "spawn_local_worker_count": -3, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 31, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 2, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 32, "async_client_threads": 31, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}' +} From 741ffb4b46891cf59d9868e79fdf82c4a8345477 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Tue, 20 Jun 2017 14:33:15 -0700 Subject: [PATCH 5/5] Rename files --- tools/jenkins/run_performance.sh | 7 +++++-- tools/jenkins/{run_performance_old.sh => run_qps_diff.sh} | 7 ++----- tools/profiling/qps/qps_diff.py | 2 +- tools/profiling/qps/qps_scenarios.py | 2 -- 4 files changed, 8 insertions(+), 10 deletions(-) rename tools/jenkins/{run_performance_old.sh => run_qps_diff.sh} (58%) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 9529b0126fc..3ce05cc7f1e 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -13,11 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# This script is invoked by Jenkins and runs a diff on the qps drivers +# This script is invoked by Jenkins and runs a diff on the microbenchmarks set -ex +# List of benchmarks that provide good signal for analyzing performance changes in pull requests +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" + # Enter the gRPC repo root cd $(dirname $0)/../.. tools/run_tests/start_port_server.py -tools/profiling/qps/qps_diff.py -d origin/$ghprbTargetBranch +tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN diff --git a/tools/jenkins/run_performance_old.sh b/tools/jenkins/run_qps_diff.sh similarity index 58% rename from tools/jenkins/run_performance_old.sh rename to tools/jenkins/run_qps_diff.sh index 3ce05cc7f1e..9529b0126fc 100755 --- a/tools/jenkins/run_performance_old.sh +++ b/tools/jenkins/run_qps_diff.sh @@ -13,14 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# This script is invoked by Jenkins and runs a diff on the microbenchmarks +# This script is invoked by Jenkins and runs a diff on the qps drivers set -ex -# List of benchmarks that provide good signal for analyzing performance changes in pull requests -BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" - # Enter the gRPC repo root cd $(dirname $0)/../.. tools/run_tests/start_port_server.py -tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b $BENCHMARKS_TO_RUN +tools/profiling/qps/qps_diff.py -d origin/$ghprbTargetBranch diff --git a/tools/profiling/qps/qps_diff.py b/tools/profiling/qps/qps_diff.py index 65c845caa1f..0654f45666f 100755 --- a/tools/profiling/qps/qps_diff.py +++ b/tools/profiling/qps/qps_diff.py @@ -118,7 +118,7 @@ def diff(scenarios, loops, old, new): for sn in scenarios: old_data[sn] = [] new_data[sn] = [] - for i in range(0, loops): + for i in range(loops): old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i))) new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i))) diff --git a/tools/profiling/qps/qps_scenarios.py b/tools/profiling/qps/qps_scenarios.py index d473d7cd240..4fbbdefc4db 100644 --- a/tools/profiling/qps/qps_scenarios.py +++ b/tools/profiling/qps/qps_scenarios.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python2.7 -# # Copyright 2017 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License");