mirror of https://github.com/grpc/grpc.git
[cleanup] Remove old microbenchmarking diff framework (#36952)
There's something new in the works, so it's time that this unmaintained & broken system got garbage collected. Closes #36952 PiperOrigin-RevId: 644184198pull/36960/head
Craig Tiller
5 months ago
committed by
Copybara-Service
parent
bc26f27c32
commit
13a8023268
16 changed files with 0 additions and 1694 deletions
@ -1,26 +0,0 @@ |
|||||||
#!/usr/bin/env bash |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
|
|
||||||
# This script is invoked by Kokoro and runs a diff on the microbenchmarks |
|
||||||
set -ex |
|
||||||
|
|
||||||
# Enter the gRPC repo root |
|
||||||
cd $(dirname $0)/../../.. |
|
||||||
|
|
||||||
source tools/internal_ci/helper_scripts/prepare_build_linux_rc |
|
||||||
|
|
||||||
export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian11_x64 |
|
||||||
export DOCKER_RUN_SCRIPT=tools/internal_ci/linux/grpc_microbenchmark_diff_in_docker.sh |
|
||||||
exec tools/run_tests/dockerize/build_and_run_docker.sh |
|
||||||
@ -1,32 +0,0 @@ |
|||||||
#!/usr/bin/env bash |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
|
|
||||||
set -ex |
|
||||||
|
|
||||||
# Enter the gRPC repo root |
|
||||||
cd $(dirname $0)/../../.. |
|
||||||
|
|
||||||
# some extra pip packages are needed for the check_on_pr.py script to work |
|
||||||
# TODO(jtattermusch): avoid needing to install these pip packages each time |
|
||||||
time python3 -m pip install --user -r tools/internal_ci/helper_scripts/requirements.linux_perf.txt |
|
||||||
|
|
||||||
# List of benchmarks that provide good signal for analyzing performance changes in pull requests |
|
||||||
BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_chttp2_hpack" |
|
||||||
|
|
||||||
tools/run_tests/start_port_server.py |
|
||||||
|
|
||||||
tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \ |
|
||||||
-d "origin/$KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH" \ |
|
||||||
-b $BENCHMARKS_TO_RUN |
|
||||||
@ -1,33 +0,0 @@ |
|||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
|
|
||||||
# Config file for the internal CI (in protobuf text format) |
|
||||||
|
|
||||||
# Location of the continuous shell script in repository. |
|
||||||
build_file: "grpc/tools/internal_ci/linux/grpc_microbenchmark_diff.sh" |
|
||||||
timeout_mins: 120 |
|
||||||
before_action { |
|
||||||
fetch_keystore { |
|
||||||
keystore_resource { |
|
||||||
keystore_config_id: 73836 |
|
||||||
keyname: "grpc_checks_private_key" |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
action { |
|
||||||
define_artifacts { |
|
||||||
regex: "**/*sponge_log.*" |
|
||||||
regex: "github/grpc/reports/**" |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,4 +0,0 @@ |
|||||||
Microbenchmarks |
|
||||||
==== |
|
||||||
|
|
||||||
This directory contains helper scripts for the microbenchmark suites. |
|
||||||
@ -1,70 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
|
|
||||||
# Convert google-benchmark json output to something that can be uploaded to |
|
||||||
# BigQuery |
|
||||||
|
|
||||||
import csv |
|
||||||
import json |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
|
|
||||||
import bm_json |
|
||||||
|
|
||||||
columns = [] |
|
||||||
|
|
||||||
for row in json.loads( |
|
||||||
# TODO(jtattermusch): make sure the dataset name is not hardcoded |
|
||||||
subprocess.check_output( |
|
||||||
["bq", "--format=json", "show", "microbenchmarks.microbenchmarks"] |
|
||||||
) |
|
||||||
)["schema"]["fields"]: |
|
||||||
columns.append((row["name"], row["type"].lower())) |
|
||||||
|
|
||||||
SANITIZE = { |
|
||||||
"integer": int, |
|
||||||
"float": float, |
|
||||||
"boolean": bool, |
|
||||||
"string": str, |
|
||||||
"timestamp": str, |
|
||||||
} |
|
||||||
|
|
||||||
# TODO(jtattermusch): add proper argparse argument, rather than trying |
|
||||||
# to emulate with manual argv inspection. |
|
||||||
if sys.argv[1] == "--schema": |
|
||||||
print(",\n".join("%s:%s" % (k, t.upper()) for k, t in columns)) |
|
||||||
sys.exit(0) |
|
||||||
|
|
||||||
with open(sys.argv[1]) as f: |
|
||||||
js = json.loads(f.read()) |
|
||||||
|
|
||||||
if len(sys.argv) > 2: |
|
||||||
with open(sys.argv[2]) as f: |
|
||||||
js2 = json.loads(f.read()) |
|
||||||
else: |
|
||||||
js2 = None |
|
||||||
|
|
||||||
# TODO(jtattermusch): write directly to a file instead of stdout |
|
||||||
writer = csv.DictWriter(sys.stdout, [c for c, t in columns]) |
|
||||||
|
|
||||||
for row in bm_json.expand_json(js, js2): |
|
||||||
sane_row = {} |
|
||||||
for name, sql_type in columns: |
|
||||||
if name in row: |
|
||||||
if row[name] == "": |
|
||||||
continue |
|
||||||
sane_row[name] = SANITIZE[sql_type](row[name]) |
|
||||||
writer.writerow(sane_row) |
|
||||||
@ -1,116 +0,0 @@ |
|||||||
The bm_diff Family |
|
||||||
==== |
|
||||||
|
|
||||||
This family of python scripts can be incredibly useful for fast iteration over |
|
||||||
different performance tweaks. The tools allow you to save performance data from |
|
||||||
a baseline commit, then quickly compare data from your working branch to that |
|
||||||
baseline data to see if you have made any performance wins. |
|
||||||
|
|
||||||
The tools operate with three concrete steps, which can be invoked separately, |
|
||||||
or all together via the driver script, bm_main.py. This readme will describe |
|
||||||
the typical workflow for these scripts, then it will include sections on the |
|
||||||
details of every script for advanced usage. |
|
||||||
|
|
||||||
## Normal Workflow |
|
||||||
|
|
||||||
Let's say you are working on a performance optimization for grpc_error. You have |
|
||||||
made some significant changes and want to see some data. From your branch, run |
|
||||||
(ensure everything is committed first): |
|
||||||
|
|
||||||
`tools/profiling/microbenchmarks/bm_diff/bm_main.py -b bm_error -l 5 -d master` |
|
||||||
|
|
||||||
This will build the `bm_error` binary on your branch, and then it will checkout |
|
||||||
master and build it there too. It will then run these benchmarks 5 times each. |
|
||||||
Lastly it will compute the statistically significant performance differences |
|
||||||
between the two branches. This should show the nice performance wins your |
|
||||||
changes have made. |
|
||||||
|
|
||||||
If you have already invoked bm_main with `-d master`, you should instead use |
|
||||||
`-o` for subsequent runs. This allows the script to skip re-building and |
|
||||||
re-running the unchanged master branch. For example: |
|
||||||
|
|
||||||
`tools/profiling/microbenchmarks/bm_diff/bm_main.py -b bm_error -l 5 -o` |
|
||||||
|
|
||||||
This will only build and run `bm_error` on your branch. It will then compare |
|
||||||
the output to the saved runs from master. |
|
||||||
|
|
||||||
## Advanced Workflow |
|
||||||
|
|
||||||
If you have a deeper knowledge of these scripts, you can use them to do more |
|
||||||
fine tuned benchmark comparisons. For example, you could build, run, and save |
|
||||||
the benchmark output from two different base branches. Then you could diff both |
|
||||||
of these baselines against your working branch to see how the different metrics |
|
||||||
change. The rest of this doc goes over the details of what each of the |
|
||||||
individual modules accomplishes. |
|
||||||
|
|
||||||
## bm_build.py |
|
||||||
|
|
||||||
This scrips builds the benchmarks. It takes in a name parameter, and will |
|
||||||
store the binaries based on that. Both `opt` and `counter` configurations |
|
||||||
will be used. The `opt` is used to get cpu_time and real_time, and the |
|
||||||
`counters` build is used to track other metrics like allocs, atomic adds, |
|
||||||
etc etc etc. |
|
||||||
|
|
||||||
For example, if you were to invoke (we assume everything is run from the |
|
||||||
root of the repo): |
|
||||||
|
|
||||||
`tools/profiling/microbenchmarks/bm_diff/bm_build.py -b bm_error -n baseline` |
|
||||||
|
|
||||||
then the microbenchmark binaries will show up under |
|
||||||
`bm_diff_baseline/{opt,counters}/bm_error` |
|
||||||
|
|
||||||
## bm_run.py |
|
||||||
|
|
||||||
This script runs the benchmarks. It takes a name parameter that must match the |
|
||||||
name that was passed to `bm_build.py`. The script then runs the benchmark |
|
||||||
multiple times (default is 20, can be toggled via the loops parameter). The |
|
||||||
output is saved as `<benchmark name>.<config>.<name>.<loop idx>.json` |
|
||||||
|
|
||||||
For example, if you were to run: |
|
||||||
|
|
||||||
`tools/profiling/microbenchmarks/bm_diff/bm_run.py -b bm_error -b baseline -l 5` |
|
||||||
|
|
||||||
Then an example output file would be `bm_error.opt.baseline.0.json` |
|
||||||
|
|
||||||
## bm_diff.py |
|
||||||
|
|
||||||
This script takes in the output from two benchmark runs, computes the diff |
|
||||||
between them, and prints any significant improvements or regressions. It takes |
|
||||||
in two name parameters, old and new. These must have previously been built and |
|
||||||
run. |
|
||||||
|
|
||||||
For example, assuming you had already built and run a 'baseline' microbenchmark |
|
||||||
from master, and then you also built and ran a 'current' microbenchmark from |
|
||||||
the branch you were working on, you could invoke: |
|
||||||
|
|
||||||
`tools/profiling/microbenchmarks/bm_diff/bm_diff.py -b bm_error -o baseline -n current -l 5` |
|
||||||
|
|
||||||
This would output the percent difference between your branch and master. |
|
||||||
|
|
||||||
## bm_main.py |
|
||||||
|
|
||||||
This is the driver script. It uses the previous three modules and does |
|
||||||
everything for you. You pass in the benchmarks to be run, the number of loops, |
|
||||||
number of CPUs to use, and the commit to compare to. Then the script will: |
|
||||||
* Build the benchmarks at head, then checkout the branch to compare to and |
|
||||||
build the benchmarks there |
|
||||||
* Run both sets of microbenchmarks |
|
||||||
* Run bm_diff.py to compare the two, outputs the difference. |
|
||||||
|
|
||||||
For example, one might run: |
|
||||||
|
|
||||||
`tools/profiling/microbenchmarks/bm_diff/bm_main.py -b bm_error -l 5 -d master` |
|
||||||
|
|
||||||
This would compare the current branch's error benchmarks to master. |
|
||||||
|
|
||||||
This script is invoked by our infrastructure on every PR to protect against |
|
||||||
regressions and demonstrate performance wins. |
|
||||||
|
|
||||||
However, if you are iterating over different performance tweaks quickly, it is |
|
||||||
unnecessary to build and run the baseline commit every time. That is why we |
|
||||||
provide a different flag in case you are sure that the baseline benchmark has |
|
||||||
already been built and run. In that case use the --old flag to pass in the name |
|
||||||
of the baseline. This will only build and run the current branch. For example: |
|
||||||
|
|
||||||
`tools/profiling/microbenchmarks/bm_diff/bm_main.py -b bm_error -l 5 -o old` |
|
||||||
|
|
||||||
@ -1,98 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
""" Python utility to build opt and counters benchmarks """ |
|
||||||
|
|
||||||
import argparse |
|
||||||
import multiprocessing |
|
||||||
import os |
|
||||||
import shutil |
|
||||||
import subprocess |
|
||||||
|
|
||||||
import bm_constants |
|
||||||
|
|
||||||
|
|
||||||
def _args(): |
|
||||||
argp = argparse.ArgumentParser(description="Builds microbenchmarks") |
|
||||||
argp.add_argument( |
|
||||||
"-b", |
|
||||||
"--benchmarks", |
|
||||||
nargs="+", |
|
||||||
choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
default=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
help="Which benchmarks to build", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-j", |
|
||||||
"--jobs", |
|
||||||
type=int, |
|
||||||
default=multiprocessing.cpu_count(), |
|
||||||
help=( |
|
||||||
"Deprecated. Bazel chooses number of CPUs to build with" |
|
||||||
" automatically." |
|
||||||
), |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-n", |
|
||||||
"--name", |
|
||||||
type=str, |
|
||||||
help=( |
|
||||||
"Unique name of this build. To be used as a handle to pass to the" |
|
||||||
" other bm* scripts" |
|
||||||
), |
|
||||||
) |
|
||||||
args = argp.parse_args() |
|
||||||
assert args.name |
|
||||||
return args |
|
||||||
|
|
||||||
|
|
||||||
def _build_cmd(cfg, benchmarks): |
|
||||||
bazel_targets = [ |
|
||||||
"//test/cpp/microbenchmarks:%s" % benchmark for benchmark in benchmarks |
|
||||||
] |
|
||||||
# --dynamic_mode=off makes sure that we get a monolithic binary that can be safely |
|
||||||
# moved outside of the bazel-bin directory |
|
||||||
return [ |
|
||||||
"tools/bazel", |
|
||||||
"build", |
|
||||||
"--config=%s" % cfg, |
|
||||||
"--dynamic_mode=off", |
|
||||||
] + bazel_targets |
|
||||||
|
|
||||||
|
|
||||||
def _build_config_and_copy(cfg, benchmarks, dest_dir): |
|
||||||
"""Build given config and copy resulting binaries to dest_dir/CONFIG""" |
|
||||||
subprocess.check_call(_build_cmd(cfg, benchmarks)) |
|
||||||
cfg_dir = dest_dir + "/%s" % cfg |
|
||||||
os.makedirs(cfg_dir) |
|
||||||
subprocess.check_call( |
|
||||||
["cp"] |
|
||||||
+ [ |
|
||||||
"bazel-bin/test/cpp/microbenchmarks/%s" % benchmark |
|
||||||
for benchmark in benchmarks |
|
||||||
] |
|
||||||
+ [cfg_dir] |
|
||||||
) |
|
||||||
|
|
||||||
|
|
||||||
def build(name, benchmarks, jobs): |
|
||||||
dest_dir = "bm_diff_%s" % name |
|
||||||
shutil.rmtree(dest_dir, ignore_errors=True) |
|
||||||
_build_config_and_copy("opt", benchmarks, dest_dir) |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
args = _args() |
|
||||||
build(args.name, args.benchmarks, args.jobs) |
|
||||||
@ -1,41 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
""" Configurable constants for the bm_*.py family """ |
|
||||||
|
|
||||||
_AVAILABLE_BENCHMARK_TESTS = [ |
|
||||||
"bm_fullstack_unary_ping_pong", |
|
||||||
"bm_fullstack_streaming_ping_pong", |
|
||||||
"bm_fullstack_streaming_pump", |
|
||||||
"bm_closure", |
|
||||||
"bm_cq", |
|
||||||
"bm_chttp2_hpack", |
|
||||||
] |
|
||||||
|
|
||||||
_INTERESTING = ( |
|
||||||
"cpu_time", |
|
||||||
"real_time", |
|
||||||
"locks_per_iteration", |
|
||||||
"allocs_per_iteration", |
|
||||||
"writes_per_iteration", |
|
||||||
"atm_cas_per_iteration", |
|
||||||
"atm_add_per_iteration", |
|
||||||
"nows_per_iteration", |
|
||||||
"cli_transport_stalls_per_iteration", |
|
||||||
"cli_stream_stalls_per_iteration", |
|
||||||
"svr_transport_stalls_per_iteration", |
|
||||||
"svr_stream_stalls_per_iteration", |
|
||||||
"http2_pings_sent_per_iteration", |
|
||||||
) |
|
||||||
@ -1,300 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
""" Computes the diff between two bm runs and outputs significant results """ |
|
||||||
|
|
||||||
import argparse |
|
||||||
import collections |
|
||||||
import json |
|
||||||
import os |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), "..")) |
|
||||||
|
|
||||||
import bm_constants |
|
||||||
import bm_json |
|
||||||
import bm_speedup |
|
||||||
import tabulate |
|
||||||
|
|
||||||
verbose = False |
|
||||||
|
|
||||||
|
|
||||||
def _median(ary): |
|
||||||
assert len(ary) |
|
||||||
ary = sorted(ary) |
|
||||||
n = len(ary) |
|
||||||
if n % 2 == 0: |
|
||||||
return (ary[(n - 1) // 2] + ary[(n - 1) // 2 + 1]) / 2.0 |
|
||||||
else: |
|
||||||
return ary[n // 2] |
|
||||||
|
|
||||||
|
|
||||||
def _args(): |
|
||||||
argp = argparse.ArgumentParser( |
|
||||||
description="Perform diff on microbenchmarks" |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-t", |
|
||||||
"--track", |
|
||||||
choices=sorted(bm_constants._INTERESTING), |
|
||||||
nargs="+", |
|
||||||
default=sorted(bm_constants._INTERESTING), |
|
||||||
help="Which metrics to track", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-b", |
|
||||||
"--benchmarks", |
|
||||||
nargs="+", |
|
||||||
choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
default=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
help="Which benchmarks to run", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-l", |
|
||||||
"--loops", |
|
||||||
type=int, |
|
||||||
default=20, |
|
||||||
help=( |
|
||||||
"Number of times to loops the benchmarks. Must match what was" |
|
||||||
" passed to bm_run.py" |
|
||||||
), |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-r", |
|
||||||
"--regex", |
|
||||||
type=str, |
|
||||||
default="", |
|
||||||
help="Regex to filter benchmarks run", |
|
||||||
) |
|
||||||
argp.add_argument("-n", "--new", type=str, help="New benchmark name") |
|
||||||
argp.add_argument("-o", "--old", type=str, help="Old benchmark name") |
|
||||||
argp.add_argument( |
|
||||||
"-v", "--verbose", type=bool, help="Print details of before/after" |
|
||||||
) |
|
||||||
args = argp.parse_args() |
|
||||||
global verbose |
|
||||||
if args.verbose: |
|
||||||
verbose = True |
|
||||||
assert args.new |
|
||||||
assert args.old |
|
||||||
return args |
|
||||||
|
|
||||||
|
|
||||||
def _maybe_print(str): |
|
||||||
if verbose: |
|
||||||
print(str) |
|
||||||
|
|
||||||
|
|
||||||
class Benchmark: |
|
||||||
def __init__(self): |
|
||||||
self.samples = { |
|
||||||
True: collections.defaultdict(list), |
|
||||||
False: collections.defaultdict(list), |
|
||||||
} |
|
||||||
self.final = {} |
|
||||||
self.speedup = {} |
|
||||||
|
|
||||||
def add_sample(self, track, data, new): |
|
||||||
for f in track: |
|
||||||
if f in data: |
|
||||||
self.samples[new][f].append(float(data[f])) |
|
||||||
|
|
||||||
def process(self, track, new_name, old_name): |
|
||||||
for f in sorted(track): |
|
||||||
new = self.samples[True][f] |
|
||||||
old = self.samples[False][f] |
|
||||||
if not new or not old: |
|
||||||
continue |
|
||||||
mdn_diff = abs(_median(new) - _median(old)) |
|
||||||
_maybe_print( |
|
||||||
"%s: %s=%r %s=%r mdn_diff=%r" |
|
||||||
% (f, new_name, new, old_name, old, mdn_diff) |
|
||||||
) |
|
||||||
s = bm_speedup.speedup(new, old, 1e-5) |
|
||||||
self.speedup[f] = s |
|
||||||
if abs(s) > 3: |
|
||||||
if mdn_diff > 0.5: |
|
||||||
self.final[f] = "%+d%%" % s |
|
||||||
return self.final.keys() |
|
||||||
|
|
||||||
def skip(self): |
|
||||||
return not self.final |
|
||||||
|
|
||||||
def row(self, flds): |
|
||||||
return [self.final[f] if f in self.final else "" for f in flds] |
|
||||||
|
|
||||||
def speedup(self, name): |
|
||||||
if name in self.speedup: |
|
||||||
return self.speedup[name] |
|
||||||
return None |
|
||||||
|
|
||||||
|
|
||||||
def _read_json(filename, badjson_files, nonexistant_files): |
|
||||||
stripped = ".".join(filename.split(".")[:-2]) |
|
||||||
try: |
|
||||||
with open(filename) as f: |
|
||||||
r = f.read() |
|
||||||
return json.loads(r) |
|
||||||
except IOError as e: |
|
||||||
if stripped in nonexistant_files: |
|
||||||
nonexistant_files[stripped] += 1 |
|
||||||
else: |
|
||||||
nonexistant_files[stripped] = 1 |
|
||||||
return None |
|
||||||
except ValueError as e: |
|
||||||
print(r) |
|
||||||
if stripped in badjson_files: |
|
||||||
badjson_files[stripped] += 1 |
|
||||||
else: |
|
||||||
badjson_files[stripped] = 1 |
|
||||||
return None |
|
||||||
|
|
||||||
|
|
||||||
def fmt_dict(d): |
|
||||||
return "".join([" " + k + ": " + str(d[k]) + "\n" for k in d]) |
|
||||||
|
|
||||||
|
|
||||||
def diff(bms, loops, regex, track, old, new): |
|
||||||
benchmarks = collections.defaultdict(Benchmark) |
|
||||||
|
|
||||||
badjson_files = {} |
|
||||||
nonexistant_files = {} |
|
||||||
for bm in bms: |
|
||||||
for loop in range(0, loops): |
|
||||||
for line in subprocess.check_output( |
|
||||||
[ |
|
||||||
"bm_diff_%s/opt/%s" % (old, bm), |
|
||||||
"--benchmark_list_tests", |
|
||||||
"--benchmark_filter=%s" % regex, |
|
||||||
] |
|
||||||
).splitlines(): |
|
||||||
line = line.decode("UTF-8") |
|
||||||
stripped_line = ( |
|
||||||
line.strip() |
|
||||||
.replace("/", "_") |
|
||||||
.replace("<", "_") |
|
||||||
.replace(">", "_") |
|
||||||
.replace(", ", "_") |
|
||||||
) |
|
||||||
js_new_opt = _read_json( |
|
||||||
"%s.%s.opt.%s.%d.json" % (bm, stripped_line, new, loop), |
|
||||||
badjson_files, |
|
||||||
nonexistant_files, |
|
||||||
) |
|
||||||
js_old_opt = _read_json( |
|
||||||
"%s.%s.opt.%s.%d.json" % (bm, stripped_line, old, loop), |
|
||||||
badjson_files, |
|
||||||
nonexistant_files, |
|
||||||
) |
|
||||||
if js_new_opt: |
|
||||||
for row in bm_json.expand_json(js_new_opt): |
|
||||||
name = row["cpp_name"] |
|
||||||
if name.endswith("_mean") or name.endswith("_stddev"): |
|
||||||
continue |
|
||||||
benchmarks[name].add_sample(track, row, True) |
|
||||||
if js_old_opt: |
|
||||||
for row in bm_json.expand_json(js_old_opt): |
|
||||||
name = row["cpp_name"] |
|
||||||
if name.endswith("_mean") or name.endswith("_stddev"): |
|
||||||
continue |
|
||||||
benchmarks[name].add_sample(track, row, False) |
|
||||||
|
|
||||||
really_interesting = set() |
|
||||||
for name, bm in benchmarks.items(): |
|
||||||
_maybe_print(name) |
|
||||||
really_interesting.update(bm.process(track, new, old)) |
|
||||||
fields = [f for f in track if f in really_interesting] |
|
||||||
|
|
||||||
# figure out the significance of the changes... right now we take the 95%-ile |
|
||||||
# benchmark delta %-age, and then apply some hand chosen thresholds |
|
||||||
histogram = [] |
|
||||||
_NOISY = ["BM_WellFlushed"] |
|
||||||
for name, bm in benchmarks.items(): |
|
||||||
if name in _NOISY: |
|
||||||
print( |
|
||||||
"skipping noisy benchmark '%s' for labelling evaluation" % name |
|
||||||
) |
|
||||||
if bm.skip(): |
|
||||||
continue |
|
||||||
d = bm.speedup["cpu_time"] |
|
||||||
if d is None: |
|
||||||
continue |
|
||||||
histogram.append(d) |
|
||||||
histogram.sort() |
|
||||||
print("histogram of speedups: ", histogram) |
|
||||||
if len(histogram) == 0: |
|
||||||
significance = 0 |
|
||||||
else: |
|
||||||
delta = histogram[int(len(histogram) * 0.95)] |
|
||||||
mul = 1 |
|
||||||
if delta < 0: |
|
||||||
delta = -delta |
|
||||||
mul = -1 |
|
||||||
if delta < 2: |
|
||||||
significance = 0 |
|
||||||
elif delta < 5: |
|
||||||
significance = 1 |
|
||||||
elif delta < 10: |
|
||||||
significance = 2 |
|
||||||
else: |
|
||||||
significance = 3 |
|
||||||
significance *= mul |
|
||||||
|
|
||||||
headers = ["Benchmark"] + fields |
|
||||||
rows = [] |
|
||||||
for name in sorted(benchmarks.keys()): |
|
||||||
if benchmarks[name].skip(): |
|
||||||
continue |
|
||||||
rows.append([name] + benchmarks[name].row(fields)) |
|
||||||
note = None |
|
||||||
if len(badjson_files): |
|
||||||
note = ( |
|
||||||
"Corrupt JSON data (indicates timeout or crash): \n%s" |
|
||||||
% fmt_dict(badjson_files) |
|
||||||
) |
|
||||||
if len(nonexistant_files): |
|
||||||
if note: |
|
||||||
note += ( |
|
||||||
"\n\nMissing files (indicates new benchmark): \n%s" |
|
||||||
% fmt_dict(nonexistant_files) |
|
||||||
) |
|
||||||
else: |
|
||||||
note = ( |
|
||||||
"\n\nMissing files (indicates new benchmark): \n%s" |
|
||||||
% fmt_dict(nonexistant_files) |
|
||||||
) |
|
||||||
if rows: |
|
||||||
return ( |
|
||||||
tabulate.tabulate(rows, headers=headers, floatfmt="+.2f"), |
|
||||||
note, |
|
||||||
significance, |
|
||||||
) |
|
||||||
else: |
|
||||||
return None, note, 0 |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
args = _args() |
|
||||||
diff, note = diff( |
|
||||||
args.benchmarks, |
|
||||||
args.loops, |
|
||||||
args.regex, |
|
||||||
args.track, |
|
||||||
args.old, |
|
||||||
args.new, |
|
||||||
args.counters, |
|
||||||
) |
|
||||||
print("%s\n%s" % (note, diff if diff else "No performance differences")) |
|
||||||
@ -1,182 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
""" Runs the entire bm_*.py pipeline, and possible comments on the PR """ |
|
||||||
|
|
||||||
import argparse |
|
||||||
import multiprocessing |
|
||||||
import os |
|
||||||
import random |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
|
|
||||||
sys.path.append( |
|
||||||
os.path.join( |
|
||||||
os.path.dirname(sys.argv[0]), "..", "..", "run_tests", "python_utils" |
|
||||||
) |
|
||||||
) |
|
||||||
|
|
||||||
sys.path.append( |
|
||||||
os.path.join( |
|
||||||
os.path.dirname(sys.argv[0]), |
|
||||||
"..", |
|
||||||
"..", |
|
||||||
"..", |
|
||||||
"run_tests", |
|
||||||
"python_utils", |
|
||||||
) |
|
||||||
) |
|
||||||
|
|
||||||
import bm_build |
|
||||||
import bm_constants |
|
||||||
import bm_diff |
|
||||||
import bm_run |
|
||||||
import check_on_pr |
|
||||||
import jobset |
|
||||||
|
|
||||||
|
|
||||||
def _args(): |
|
||||||
argp = argparse.ArgumentParser( |
|
||||||
description="Perform diff on microbenchmarks" |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-t", |
|
||||||
"--track", |
|
||||||
choices=sorted(bm_constants._INTERESTING), |
|
||||||
nargs="+", |
|
||||||
default=sorted(bm_constants._INTERESTING), |
|
||||||
help="Which metrics to track", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-b", |
|
||||||
"--benchmarks", |
|
||||||
nargs="+", |
|
||||||
choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
default=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
help="Which benchmarks to run", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-d", |
|
||||||
"--diff_base", |
|
||||||
type=str, |
|
||||||
help="Commit or branch to compare the current one to", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-o", |
|
||||||
"--old", |
|
||||||
default="old", |
|
||||||
type=str, |
|
||||||
help='Name of baseline run to compare to. Usually just called "old"', |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-r", |
|
||||||
"--regex", |
|
||||||
type=str, |
|
||||||
default="", |
|
||||||
help="Regex to filter benchmarks run", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-l", |
|
||||||
"--loops", |
|
||||||
type=int, |
|
||||||
default=10, |
|
||||||
help=( |
|
||||||
"Number of times to loops the benchmarks. More loops cuts down on" |
|
||||||
" noise" |
|
||||||
), |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-j", |
|
||||||
"--jobs", |
|
||||||
type=int, |
|
||||||
default=multiprocessing.cpu_count(), |
|
||||||
help="Number of CPUs to use", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"--pr_comment_name", |
|
||||||
type=str, |
|
||||||
default="microbenchmarks", |
|
||||||
help="Name that Jenkins will use to comment on the PR", |
|
||||||
) |
|
||||||
args = argp.parse_args() |
|
||||||
assert args.diff_base or args.old, "One of diff_base or old must be set!" |
|
||||||
if args.loops < 3: |
|
||||||
print("WARNING: This run will likely be noisy. Increase loops.") |
|
||||||
return args |
|
||||||
|
|
||||||
|
|
||||||
def eintr_be_gone(fn): |
|
||||||
"""Run fn until it doesn't stop because of EINTR""" |
|
||||||
|
|
||||||
def inner(*args): |
|
||||||
while True: |
|
||||||
try: |
|
||||||
return fn(*args) |
|
||||||
except IOError as e: |
|
||||||
if e.errno != errno.EINTR: |
|
||||||
raise |
|
||||||
|
|
||||||
return inner |
|
||||||
|
|
||||||
|
|
||||||
def main(args): |
|
||||||
bm_build.build("new", args.benchmarks, args.jobs) |
|
||||||
|
|
||||||
old = args.old |
|
||||||
if args.diff_base: |
|
||||||
old = "old" |
|
||||||
where_am_i = subprocess.check_output( |
|
||||||
["git", "rev-parse", "--abbrev-ref", "HEAD"] |
|
||||||
).strip() |
|
||||||
subprocess.check_call(["git", "checkout", args.diff_base]) |
|
||||||
try: |
|
||||||
bm_build.build(old, args.benchmarks, args.jobs) |
|
||||||
finally: |
|
||||||
subprocess.check_call(["git", "checkout", where_am_i]) |
|
||||||
subprocess.check_call(["git", "submodule", "update"]) |
|
||||||
|
|
||||||
jobs_list = [] |
|
||||||
jobs_list += bm_run.create_jobs( |
|
||||||
"new", args.benchmarks, args.loops, args.regex |
|
||||||
) |
|
||||||
jobs_list += bm_run.create_jobs( |
|
||||||
old, args.benchmarks, args.loops, args.regex |
|
||||||
) |
|
||||||
|
|
||||||
# shuffle all jobs to eliminate noise from GCE CPU drift |
|
||||||
random.shuffle(jobs_list, random.SystemRandom().random) |
|
||||||
jobset.run(jobs_list, maxjobs=args.jobs) |
|
||||||
|
|
||||||
diff, note, significance = bm_diff.diff( |
|
||||||
args.benchmarks, args.loops, args.regex, args.track, old, "new" |
|
||||||
) |
|
||||||
if diff: |
|
||||||
text = "[%s] Performance differences noted:\n%s" % ( |
|
||||||
args.pr_comment_name, |
|
||||||
diff, |
|
||||||
) |
|
||||||
else: |
|
||||||
text = ( |
|
||||||
"[%s] No significant performance differences" % args.pr_comment_name |
|
||||||
) |
|
||||||
if note: |
|
||||||
text = note + "\n\n" + text |
|
||||||
print("%s" % text) |
|
||||||
check_on_pr.check_on_pr("Benchmark", "```\n%s\n```" % text) |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
args = _args() |
|
||||||
main(args) |
|
||||||
@ -1,148 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
""" Python utility to run opt and counters benchmarks and save json output """ |
|
||||||
|
|
||||||
import argparse |
|
||||||
import itertools |
|
||||||
import multiprocessing |
|
||||||
import os |
|
||||||
import random |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
|
|
||||||
import bm_constants |
|
||||||
import jobset |
|
||||||
|
|
||||||
sys.path.append( |
|
||||||
os.path.join( |
|
||||||
os.path.dirname(sys.argv[0]), |
|
||||||
"..", |
|
||||||
"..", |
|
||||||
"..", |
|
||||||
"run_tests", |
|
||||||
"python_utils", |
|
||||||
) |
|
||||||
) |
|
||||||
|
|
||||||
|
|
||||||
def _args(): |
|
||||||
argp = argparse.ArgumentParser(description="Runs microbenchmarks") |
|
||||||
argp.add_argument( |
|
||||||
"-b", |
|
||||||
"--benchmarks", |
|
||||||
nargs="+", |
|
||||||
choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
default=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
help="Benchmarks to run", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-j", |
|
||||||
"--jobs", |
|
||||||
type=int, |
|
||||||
default=multiprocessing.cpu_count(), |
|
||||||
help="Number of CPUs to use", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-n", |
|
||||||
"--name", |
|
||||||
type=str, |
|
||||||
help=( |
|
||||||
"Unique name of the build to run. Needs to match the handle passed" |
|
||||||
" to bm_build.py" |
|
||||||
), |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-r", |
|
||||||
"--regex", |
|
||||||
type=str, |
|
||||||
default="", |
|
||||||
help="Regex to filter benchmarks run", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-l", |
|
||||||
"--loops", |
|
||||||
type=int, |
|
||||||
default=20, |
|
||||||
help=( |
|
||||||
"Number of times to loops the benchmarks. More loops cuts down on" |
|
||||||
" noise" |
|
||||||
), |
|
||||||
) |
|
||||||
argp.add_argument("--counters", dest="counters", action="store_true") |
|
||||||
argp.add_argument("--no-counters", dest="counters", action="store_false") |
|
||||||
argp.set_defaults(counters=True) |
|
||||||
args = argp.parse_args() |
|
||||||
assert args.name |
|
||||||
if args.loops < 3: |
|
||||||
print( |
|
||||||
"WARNING: This run will likely be noisy. Increase loops to at " |
|
||||||
"least 3." |
|
||||||
) |
|
||||||
return args |
|
||||||
|
|
||||||
|
|
||||||
def _collect_bm_data(bm, cfg, name, regex, idx, loops): |
|
||||||
jobs_list = [] |
|
||||||
for line in subprocess.check_output( |
|
||||||
[ |
|
||||||
"bm_diff_%s/%s/%s" % (name, cfg, bm), |
|
||||||
"--benchmark_list_tests", |
|
||||||
"--benchmark_filter=%s" % regex, |
|
||||||
] |
|
||||||
).splitlines(): |
|
||||||
line = line.decode("UTF-8") |
|
||||||
stripped_line = ( |
|
||||||
line.strip() |
|
||||||
.replace("/", "_") |
|
||||||
.replace("<", "_") |
|
||||||
.replace(">", "_") |
|
||||||
.replace(", ", "_") |
|
||||||
) |
|
||||||
cmd = [ |
|
||||||
"bm_diff_%s/%s/%s" % (name, cfg, bm), |
|
||||||
"--benchmark_filter=^%s$" % line, |
|
||||||
"--benchmark_out=%s.%s.%s.%s.%d.json" |
|
||||||
% (bm, stripped_line, cfg, name, idx), |
|
||||||
"--benchmark_out_format=json", |
|
||||||
] |
|
||||||
jobs_list.append( |
|
||||||
jobset.JobSpec( |
|
||||||
cmd, |
|
||||||
shortname="%s %s %s %s %d/%d" |
|
||||||
% (bm, line, cfg, name, idx + 1, loops), |
|
||||||
verbose_success=True, |
|
||||||
cpu_cost=2, |
|
||||||
timeout_seconds=60 * 60, |
|
||||||
) |
|
||||||
) # one hour |
|
||||||
return jobs_list |
|
||||||
|
|
||||||
|
|
||||||
def create_jobs(name, benchmarks, loops, regex): |
|
||||||
jobs_list = [] |
|
||||||
for loop in range(0, loops): |
|
||||||
for bm in benchmarks: |
|
||||||
jobs_list += _collect_bm_data(bm, "opt", name, regex, loop, loops) |
|
||||||
random.shuffle(jobs_list, random.SystemRandom().random) |
|
||||||
return jobs_list |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
args = _args() |
|
||||||
jobs_list = create_jobs( |
|
||||||
args.name, args.benchmarks, args.loops, args.regex, args.counters |
|
||||||
) |
|
||||||
jobset.run(jobs_list, maxjobs=args.jobs) |
|
||||||
@ -1,68 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
|
|
||||||
import math |
|
||||||
|
|
||||||
from scipy import stats |
|
||||||
|
|
||||||
_DEFAULT_THRESHOLD = 1e-10 |
|
||||||
|
|
||||||
|
|
||||||
def scale(a, mul): |
|
||||||
return [x * mul for x in a] |
|
||||||
|
|
||||||
|
|
||||||
def cmp(a, b): |
|
||||||
return stats.ttest_ind(a, b) |
|
||||||
|
|
||||||
|
|
||||||
def speedup(new, old, threshold=_DEFAULT_THRESHOLD): |
|
||||||
if (len(set(new))) == 1 and new == old: |
|
||||||
return 0 |
|
||||||
s0, p0 = cmp(new, old) |
|
||||||
if math.isnan(p0): |
|
||||||
return 0 |
|
||||||
if s0 == 0: |
|
||||||
return 0 |
|
||||||
if p0 > threshold: |
|
||||||
return 0 |
|
||||||
if s0 < 0: |
|
||||||
pct = 1 |
|
||||||
while pct < 100: |
|
||||||
sp, pp = cmp(new, scale(old, 1 - pct / 100.0)) |
|
||||||
if sp > 0: |
|
||||||
break |
|
||||||
if pp > threshold: |
|
||||||
break |
|
||||||
pct += 1 |
|
||||||
return -(pct - 1) |
|
||||||
else: |
|
||||||
pct = 1 |
|
||||||
while pct < 10000: |
|
||||||
sp, pp = cmp(new, scale(old, 1 + pct / 100.0)) |
|
||||||
if sp < 0: |
|
||||||
break |
|
||||||
if pp > threshold: |
|
||||||
break |
|
||||||
pct += 1 |
|
||||||
return pct - 1 |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
new = [0.0, 0.0, 0.0, 0.0] |
|
||||||
old = [2.96608e-06, 3.35076e-06, 3.45384e-06, 3.34407e-06] |
|
||||||
print(speedup(new, old, 1e-5)) |
|
||||||
print(speedup(old, new, 1e-5)) |
|
||||||
@ -1,214 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
|
|
||||||
# Utilities for manipulating JSON data that represents microbenchmark results. |
|
||||||
|
|
||||||
import os |
|
||||||
|
|
||||||
# template arguments and dynamic arguments of individual benchmark types |
|
||||||
# Example benchmark name: "BM_UnaryPingPong<TCP, NoOpMutator, NoOpMutator>/0/0" |
|
||||||
_BM_SPECS = { |
|
||||||
"BM_UnaryPingPong": { |
|
||||||
"tpl": ["fixture", "client_mutator", "server_mutator"], |
|
||||||
"dyn": ["request_size", "response_size"], |
|
||||||
}, |
|
||||||
"BM_PumpStreamClientToServer": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": ["request_size"], |
|
||||||
}, |
|
||||||
"BM_PumpStreamServerToClient": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": ["request_size"], |
|
||||||
}, |
|
||||||
"BM_StreamingPingPong": { |
|
||||||
"tpl": ["fixture", "client_mutator", "server_mutator"], |
|
||||||
"dyn": ["request_size", "request_count"], |
|
||||||
}, |
|
||||||
"BM_StreamingPingPongMsgs": { |
|
||||||
"tpl": ["fixture", "client_mutator", "server_mutator"], |
|
||||||
"dyn": ["request_size"], |
|
||||||
}, |
|
||||||
"BM_PumpStreamServerToClient_Trickle": { |
|
||||||
"tpl": [], |
|
||||||
"dyn": ["request_size", "bandwidth_kilobits"], |
|
||||||
}, |
|
||||||
"BM_PumpUnbalancedUnary_Trickle": { |
|
||||||
"tpl": [], |
|
||||||
"dyn": ["cli_req_size", "svr_req_size", "bandwidth_kilobits"], |
|
||||||
}, |
|
||||||
"BM_ErrorStringOnNewError": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_ErrorStringRepeatedly": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_ErrorGetStatus": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_ErrorGetStatusCode": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_ErrorHttpError": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_HasClearGrpcStatus": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_IsolatedFilter": { |
|
||||||
"tpl": ["fixture", "client_mutator"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_HpackEncoderEncodeHeader": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": ["end_of_stream", "request_size"], |
|
||||||
}, |
|
||||||
"BM_HpackParserParseHeader": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_CallCreateDestroy": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_Zalloc": { |
|
||||||
"tpl": [], |
|
||||||
"dyn": ["request_size"], |
|
||||||
}, |
|
||||||
"BM_PollEmptyPollset_SpeedOfLight": { |
|
||||||
"tpl": [], |
|
||||||
"dyn": ["request_size", "request_count"], |
|
||||||
}, |
|
||||||
"BM_StreamCreateSendInitialMetadataDestroy": { |
|
||||||
"tpl": ["fixture"], |
|
||||||
"dyn": [], |
|
||||||
}, |
|
||||||
"BM_TransportStreamSend": { |
|
||||||
"tpl": [], |
|
||||||
"dyn": ["request_size"], |
|
||||||
}, |
|
||||||
"BM_TransportStreamRecv": { |
|
||||||
"tpl": [], |
|
||||||
"dyn": ["request_size"], |
|
||||||
}, |
|
||||||
"BM_StreamingPingPongWithCoalescingApi": { |
|
||||||
"tpl": ["fixture", "client_mutator", "server_mutator"], |
|
||||||
"dyn": ["request_size", "request_count", "end_of_stream"], |
|
||||||
}, |
|
||||||
"BM_Base16SomeStuff": { |
|
||||||
"tpl": [], |
|
||||||
"dyn": ["request_size"], |
|
||||||
}, |
|
||||||
} |
|
||||||
|
|
||||||
|
|
||||||
def numericalize(s): |
|
||||||
"""Convert abbreviations like '100M' or '10k' to a number.""" |
|
||||||
if not s: |
|
||||||
return "" |
|
||||||
if s[-1] == "k": |
|
||||||
return float(s[:-1]) * 1024 |
|
||||||
if s[-1] == "M": |
|
||||||
return float(s[:-1]) * 1024 * 1024 |
|
||||||
if 0 <= (ord(s[-1]) - ord("0")) <= 9: |
|
||||||
return float(s) |
|
||||||
assert "not a number: %s" % s |
|
||||||
|
|
||||||
|
|
||||||
def parse_name(name): |
|
||||||
cpp_name = name |
|
||||||
if "<" not in name and "/" not in name and name not in _BM_SPECS: |
|
||||||
return {"name": name, "cpp_name": name} |
|
||||||
rest = name |
|
||||||
out = {} |
|
||||||
tpl_args = [] |
|
||||||
dyn_args = [] |
|
||||||
if "<" in rest: |
|
||||||
tpl_bit = rest[rest.find("<") + 1 : rest.rfind(">")] |
|
||||||
arg = "" |
|
||||||
nesting = 0 |
|
||||||
for c in tpl_bit: |
|
||||||
if c == "<": |
|
||||||
nesting += 1 |
|
||||||
arg += c |
|
||||||
elif c == ">": |
|
||||||
nesting -= 1 |
|
||||||
arg += c |
|
||||||
elif c == ",": |
|
||||||
if nesting == 0: |
|
||||||
tpl_args.append(arg.strip()) |
|
||||||
arg = "" |
|
||||||
else: |
|
||||||
arg += c |
|
||||||
else: |
|
||||||
arg += c |
|
||||||
tpl_args.append(arg.strip()) |
|
||||||
rest = rest[: rest.find("<")] + rest[rest.rfind(">") + 1 :] |
|
||||||
if "/" in rest: |
|
||||||
s = rest.split("/") |
|
||||||
rest = s[0] |
|
||||||
dyn_args = s[1:] |
|
||||||
name = rest |
|
||||||
assert name in _BM_SPECS, "_BM_SPECS needs to be expanded for %s" % name |
|
||||||
assert len(dyn_args) == len(_BM_SPECS[name]["dyn"]) |
|
||||||
assert len(tpl_args) == len(_BM_SPECS[name]["tpl"]) |
|
||||||
out["name"] = name |
|
||||||
out["cpp_name"] = cpp_name |
|
||||||
out.update( |
|
||||||
dict( |
|
||||||
(k, numericalize(v)) |
|
||||||
for k, v in zip(_BM_SPECS[name]["dyn"], dyn_args) |
|
||||||
) |
|
||||||
) |
|
||||||
out.update(dict(zip(_BM_SPECS[name]["tpl"], tpl_args))) |
|
||||||
return out |
|
||||||
|
|
||||||
|
|
||||||
def expand_json(js): |
|
||||||
if not js: |
|
||||||
raise StopIteration() |
|
||||||
for bm in js["benchmarks"]: |
|
||||||
if bm["name"].endswith("_stddev") or bm["name"].endswith("_mean"): |
|
||||||
continue |
|
||||||
context = js["context"] |
|
||||||
if "label" in bm: |
|
||||||
labels_list = [ |
|
||||||
s.split(":") |
|
||||||
for s in bm["label"].strip().split(" ") |
|
||||||
if len(s) and s[0] != "#" |
|
||||||
] |
|
||||||
for el in labels_list: |
|
||||||
el[0] = el[0].replace("/iter", "_per_iteration") |
|
||||||
labels = dict(labels_list) |
|
||||||
else: |
|
||||||
labels = {} |
|
||||||
# TODO(jtattermusch): grabbing kokoro env values shouldn't be buried |
|
||||||
# deep in the JSON conversion logic. |
|
||||||
# Link the data to a kokoro job run by adding |
|
||||||
# well known kokoro env variables as metadata for each row |
|
||||||
row = { |
|
||||||
"jenkins_build": os.environ.get("KOKORO_BUILD_NUMBER", ""), |
|
||||||
"jenkins_job": os.environ.get("KOKORO_JOB_NAME", ""), |
|
||||||
} |
|
||||||
row.update(context) |
|
||||||
row.update(bm) |
|
||||||
row.update(parse_name(row["name"])) |
|
||||||
row.update(labels) |
|
||||||
yield row |
|
||||||
@ -1,191 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
""" Computes the diff between two qps runs and outputs significant results """ |
|
||||||
|
|
||||||
import argparse |
|
||||||
import json |
|
||||||
import multiprocessing |
|
||||||
import os |
|
||||||
import shutil |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
|
|
||||||
import qps_scenarios |
|
||||||
import tabulate |
|
||||||
|
|
||||||
sys.path.append( |
|
||||||
os.path.join( |
|
||||||
os.path.dirname(sys.argv[0]), "..", "microbenchmarks", "bm_diff" |
|
||||||
) |
|
||||||
) |
|
||||||
import bm_speedup |
|
||||||
|
|
||||||
sys.path.append( |
|
||||||
os.path.join( |
|
||||||
os.path.dirname(sys.argv[0]), "..", "..", "run_tests", "python_utils" |
|
||||||
) |
|
||||||
) |
|
||||||
import check_on_pr |
|
||||||
|
|
||||||
|
|
||||||
def _args(): |
|
||||||
argp = argparse.ArgumentParser(description="Perform diff on QPS Driver") |
|
||||||
argp.add_argument( |
|
||||||
"-d", |
|
||||||
"--diff_base", |
|
||||||
type=str, |
|
||||||
help="Commit or branch to compare the current one to", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-l", |
|
||||||
"--loops", |
|
||||||
type=int, |
|
||||||
default=4, |
|
||||||
help=( |
|
||||||
"Number of loops for each benchmark. More loops cuts down on noise" |
|
||||||
), |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-j", |
|
||||||
"--jobs", |
|
||||||
type=int, |
|
||||||
default=multiprocessing.cpu_count(), |
|
||||||
help="Number of CPUs to use", |
|
||||||
) |
|
||||||
args = argp.parse_args() |
|
||||||
assert args.diff_base, "diff_base must be set" |
|
||||||
return args |
|
||||||
|
|
||||||
|
|
||||||
def _make_cmd(jobs): |
|
||||||
return ["make", "-j", "%d" % jobs, "qps_json_driver", "qps_worker"] |
|
||||||
|
|
||||||
|
|
||||||
def build(name, jobs): |
|
||||||
shutil.rmtree("qps_diff_%s" % name, ignore_errors=True) |
|
||||||
subprocess.check_call(["git", "submodule", "update"]) |
|
||||||
try: |
|
||||||
subprocess.check_call(_make_cmd(jobs)) |
|
||||||
except subprocess.CalledProcessError as e: |
|
||||||
subprocess.check_call(["make", "clean"]) |
|
||||||
subprocess.check_call(_make_cmd(jobs)) |
|
||||||
os.rename("bins", "qps_diff_%s" % name) |
|
||||||
|
|
||||||
|
|
||||||
def _run_cmd(name, scenario, fname): |
|
||||||
return [ |
|
||||||
"qps_diff_%s/opt/qps_json_driver" % name, |
|
||||||
"--scenarios_json", |
|
||||||
scenario, |
|
||||||
"--json_file_out", |
|
||||||
fname, |
|
||||||
] |
|
||||||
|
|
||||||
|
|
||||||
def run(name, scenarios, loops): |
|
||||||
for sn in scenarios: |
|
||||||
for i in range(0, loops): |
|
||||||
fname = "%s.%s.%d.json" % (sn, name, i) |
|
||||||
subprocess.check_call(_run_cmd(name, scenarios[sn], fname)) |
|
||||||
|
|
||||||
|
|
||||||
def _load_qps(fname): |
|
||||||
try: |
|
||||||
with open(fname) as f: |
|
||||||
return json.loads(f.read())["qps"] |
|
||||||
except IOError as e: |
|
||||||
print(("IOError occurred reading file: %s" % fname)) |
|
||||||
return None |
|
||||||
except ValueError as e: |
|
||||||
print(("ValueError occurred reading file: %s" % fname)) |
|
||||||
return None |
|
||||||
|
|
||||||
|
|
||||||
def _median(ary): |
|
||||||
assert len(ary) |
|
||||||
ary = sorted(ary) |
|
||||||
n = len(ary) |
|
||||||
if n % 2 == 0: |
|
||||||
return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0 |
|
||||||
else: |
|
||||||
return ary[n / 2] |
|
||||||
|
|
||||||
|
|
||||||
def diff(scenarios, loops, old, new): |
|
||||||
old_data = {} |
|
||||||
new_data = {} |
|
||||||
|
|
||||||
# collect data |
|
||||||
for sn in scenarios: |
|
||||||
old_data[sn] = [] |
|
||||||
new_data[sn] = [] |
|
||||||
for i in range(loops): |
|
||||||
old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i))) |
|
||||||
new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i))) |
|
||||||
|
|
||||||
# crunch data |
|
||||||
headers = ["Benchmark", "qps"] |
|
||||||
rows = [] |
|
||||||
for sn in scenarios: |
|
||||||
mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn])) |
|
||||||
print( |
|
||||||
"%s: %s=%r %s=%r mdn_diff=%r" |
|
||||||
% (sn, new, new_data[sn], old, old_data[sn], mdn_diff) |
|
||||||
) |
|
||||||
s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5) |
|
||||||
if abs(s) > 3 and mdn_diff > 0.5: |
|
||||||
rows.append([sn, "%+d%%" % s]) |
|
||||||
|
|
||||||
if rows: |
|
||||||
return tabulate.tabulate(rows, headers=headers, floatfmt="+.2f") |
|
||||||
else: |
|
||||||
return None |
|
||||||
|
|
||||||
|
|
||||||
def main(args): |
|
||||||
build("new", args.jobs) |
|
||||||
|
|
||||||
if args.diff_base: |
|
||||||
where_am_i = ( |
|
||||||
subprocess.check_output( |
|
||||||
["git", "rev-parse", "--abbrev-ref", "HEAD"] |
|
||||||
) |
|
||||||
.decode() |
|
||||||
.strip() |
|
||||||
) |
|
||||||
subprocess.check_call(["git", "checkout", args.diff_base]) |
|
||||||
try: |
|
||||||
build("old", args.jobs) |
|
||||||
finally: |
|
||||||
subprocess.check_call(["git", "checkout", where_am_i]) |
|
||||||
subprocess.check_call(["git", "submodule", "update"]) |
|
||||||
|
|
||||||
run("new", qps_scenarios._SCENARIOS, args.loops) |
|
||||||
run("old", qps_scenarios._SCENARIOS, args.loops) |
|
||||||
|
|
||||||
diff_output = diff(qps_scenarios._SCENARIOS, args.loops, "old", "new") |
|
||||||
|
|
||||||
if diff_output: |
|
||||||
text = "[qps] Performance differences noted:\n%s" % diff_output |
|
||||||
else: |
|
||||||
text = "[qps] No significant performance differences" |
|
||||||
print(("%s" % text)) |
|
||||||
check_on_pr.check_on_pr("QPS", "```\n%s\n```" % text) |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
args = _args() |
|
||||||
main(args) |
|
||||||
@ -1,168 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
# Copyright 2017 gRPC authors. |
|
||||||
# |
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||||
# you may not use this file except in compliance with the License. |
|
||||||
# You may obtain a copy of the License at |
|
||||||
# |
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 |
|
||||||
# |
|
||||||
# Unless required by applicable law or agreed to in writing, software |
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, |
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||||
# See the License for the specific language governing permissions and |
|
||||||
# limitations under the License. |
|
||||||
|
|
||||||
import argparse |
|
||||||
import html |
|
||||||
import multiprocessing |
|
||||||
import os |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
|
|
||||||
import python_utils.jobset as jobset |
|
||||||
import python_utils.start_port_server as start_port_server |
|
||||||
|
|
||||||
sys.path.append( |
|
||||||
os.path.join( |
|
||||||
os.path.dirname(sys.argv[0]), |
|
||||||
"..", |
|
||||||
"profiling", |
|
||||||
"microbenchmarks", |
|
||||||
"bm_diff", |
|
||||||
) |
|
||||||
) |
|
||||||
import bm_constants |
|
||||||
|
|
||||||
flamegraph_dir = os.path.join(os.path.expanduser("~"), "FlameGraph") |
|
||||||
|
|
||||||
os.chdir(os.path.join(os.path.dirname(sys.argv[0]), "../..")) |
|
||||||
if not os.path.exists("reports"): |
|
||||||
os.makedirs("reports") |
|
||||||
|
|
||||||
start_port_server.start_port_server() |
|
||||||
|
|
||||||
|
|
||||||
def fnize(s): |
|
||||||
out = "" |
|
||||||
for c in s: |
|
||||||
if c in "<>, /": |
|
||||||
if len(out) and out[-1] == "_": |
|
||||||
continue |
|
||||||
out += "_" |
|
||||||
else: |
|
||||||
out += c |
|
||||||
return out |
|
||||||
|
|
||||||
|
|
||||||
# index html |
|
||||||
index_html = """ |
|
||||||
<html> |
|
||||||
<head> |
|
||||||
<title>Microbenchmark Results</title> |
|
||||||
</head> |
|
||||||
<body> |
|
||||||
""" |
|
||||||
|
|
||||||
|
|
||||||
def heading(name): |
|
||||||
global index_html |
|
||||||
index_html += "<h1>%s</h1>\n" % name |
|
||||||
|
|
||||||
|
|
||||||
def link(txt, tgt): |
|
||||||
global index_html |
|
||||||
index_html += '<p><a href="%s">%s</a></p>\n' % ( |
|
||||||
html.escape(tgt, quote=True), |
|
||||||
html.escape(txt), |
|
||||||
) |
|
||||||
|
|
||||||
|
|
||||||
def text(txt): |
|
||||||
global index_html |
|
||||||
index_html += "<p><pre>%s</pre></p>\n" % html.escape(txt) |
|
||||||
|
|
||||||
|
|
||||||
def _bazel_build_benchmark(bm_name, cfg): |
|
||||||
"""Build given benchmark with bazel""" |
|
||||||
subprocess.check_call( |
|
||||||
[ |
|
||||||
"tools/bazel", |
|
||||||
"build", |
|
||||||
"--config=%s" % cfg, |
|
||||||
"//test/cpp/microbenchmarks:%s" % bm_name, |
|
||||||
] |
|
||||||
) |
|
||||||
|
|
||||||
|
|
||||||
def run_summary(bm_name, cfg, base_json_name): |
|
||||||
_bazel_build_benchmark(bm_name, cfg) |
|
||||||
cmd = [ |
|
||||||
"bazel-bin/test/cpp/microbenchmarks/%s" % bm_name, |
|
||||||
"--benchmark_out=%s.%s.json" % (base_json_name, cfg), |
|
||||||
"--benchmark_out_format=json", |
|
||||||
] |
|
||||||
if args.summary_time is not None: |
|
||||||
cmd += ["--benchmark_min_time=%d" % args.summary_time] |
|
||||||
return subprocess.check_output(cmd).decode("UTF-8") |
|
||||||
|
|
||||||
|
|
||||||
def collect_summary(bm_name, args): |
|
||||||
# no counters, run microbenchmark and add summary |
|
||||||
# both to HTML report and to console. |
|
||||||
nocounters_heading = "Summary: %s" % bm_name |
|
||||||
nocounters_summary = run_summary(bm_name, "opt", bm_name) |
|
||||||
heading(nocounters_heading) |
|
||||||
text(nocounters_summary) |
|
||||||
print(nocounters_heading) |
|
||||||
print(nocounters_summary) |
|
||||||
|
|
||||||
|
|
||||||
collectors = { |
|
||||||
"summary": collect_summary, |
|
||||||
} |
|
||||||
|
|
||||||
argp = argparse.ArgumentParser(description="Collect data from microbenchmarks") |
|
||||||
argp.add_argument( |
|
||||||
"-c", |
|
||||||
"--collect", |
|
||||||
choices=sorted(collectors.keys()), |
|
||||||
nargs="*", |
|
||||||
default=sorted(collectors.keys()), |
|
||||||
help="Which collectors should be run against each benchmark", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"-b", |
|
||||||
"--benchmarks", |
|
||||||
choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
default=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
|
||||||
nargs="+", |
|
||||||
type=str, |
|
||||||
help="Which microbenchmarks should be run", |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"--bq_result_table", |
|
||||||
default="", |
|
||||||
type=str, |
|
||||||
help=( |
|
||||||
"Upload results from summary collection to a specified bigquery table." |
|
||||||
), |
|
||||||
) |
|
||||||
argp.add_argument( |
|
||||||
"--summary_time", |
|
||||||
default=None, |
|
||||||
type=int, |
|
||||||
help="Minimum time to run benchmarks for the summary collection", |
|
||||||
) |
|
||||||
args = argp.parse_args() |
|
||||||
|
|
||||||
try: |
|
||||||
for collect in args.collect: |
|
||||||
for bm_name in args.benchmarks: |
|
||||||
collectors[collect](bm_name, args) |
|
||||||
finally: |
|
||||||
if not os.path.exists("reports"): |
|
||||||
os.makedirs("reports") |
|
||||||
index_html += "</body>\n</html>\n" |
|
||||||
with open("reports/index.html", "w") as f: |
|
||||||
f.write(index_html) |
|
||||||
Loading…
Reference in new issue