Migrate the bm_diff benchmarks to python3 (#25619)

* Migrate the bm_diff benchmarks to python3

Includes a requirements.txt pinned at ~2017 versions, when this script
was first written.

* Replace p2 with p3 scipy/numpy dependencies.

* py2->3 for benchmark setup scripts

* upgrade pip to resolve python3 cryptography/setuptools-rust problem

* re-add jobset import (accidentally removed)

* re-add six's urllib import. This file is still used in py2 tests

* force py3 in run_if_c_cpp_modified.sh

* Fix another instance of subprocess.check_output binary mode

* Use the requirements.txt for CI perf environment setup

* Try to upgrade PyJWT. (v2.0.0 was problematic, #25053)

v2.x makes encode return strs from jwt.encode in both py2 and py3.
Previously, py3 would return bytes, and py2 a str.

* upgate cryptography lib version requirements for jwt.

* Wrap pip requirements specifier in quotes '>=x,<y'

* Decode subprocess output once instead of for every line

* Revert "Decode subprocess output once instead of for every line"

This reverts commit 28d14026431622ac7afd3535a8d7118e5be96628.

py2 doesn't support the `text` argument to subprocess.check_output.

* Address reviewer requests

* Pin a valid scipy version

* Remove scipy and tabulate dependencies from macos tests
pull/25117/head
AJ Heller 4 years ago committed by GitHub
parent 5c95dd5c2c
commit 5139a012e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      tools/gce/linux_kokoro_performance_worker_init.sh
  2. 5
      tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc
  3. 7
      tools/internal_ci/helper_scripts/prepare_build_macos_rc
  4. 5
      tools/internal_ci/helper_scripts/requirements.linux_perf.txt
  5. 5
      tools/internal_ci/helper_scripts/requirements.macos.txt
  6. 2
      tools/internal_ci/linux/grpc_performance_profile_daily.sh
  7. 2
      tools/internal_ci/linux/run_if_c_cpp_modified.sh
  8. 2
      tools/profiling/bloat/bloat_diff.py
  9. 4
      tools/profiling/microbenchmarks/bm2bq.py
  10. 8
      tools/profiling/microbenchmarks/bm_diff/bm_build.py
  11. 2
      tools/profiling/microbenchmarks/bm_diff/bm_constants.py
  12. 32
      tools/profiling/microbenchmarks/bm_diff/bm_diff.py
  13. 20
      tools/profiling/microbenchmarks/bm_diff/bm_main.py
  14. 18
      tools/profiling/microbenchmarks/bm_diff/bm_run.py
  15. 9
      tools/profiling/microbenchmarks/bm_diff/bm_speedup.py
  16. 1
      tools/profiling/microbenchmarks/bm_json.py
  17. 2
      tools/run_tests/python_utils/check_on_pr.py
  18. 11
      tools/run_tests/python_utils/filter_pull_request_tests.py
  19. 1
      tools/run_tests/python_utils/start_port_server.py
  20. 4
      tools/run_tests/run_microbenchmark.py
  21. 3
      tools/run_tests/start_port_server.py

@ -200,7 +200,7 @@ echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb
git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph
# Install scipy and numpy for benchmarking scripts
sudo apt-get install -y python-scipy python-numpy
sudo apt-get install -y python3-scipy python3-numpy
# Install docker
curl -sSL https://get.docker.com/ | sh

@ -19,9 +19,10 @@
ulimit -n 32768
ulimit -c unlimited
sudo pip install tabulate
python3 -m pip install pip==19.3.1
# Python dependencies for tools/run_tests/python_utils/check_on_pr.py
time python2.7 -m pip install pyjwt cryptography requests --user
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
time python3 -m pip install --user -r $DIR/requirements.linux_perf.txt
git submodule update --init

@ -37,6 +37,7 @@ brew config
# Add GCP credentials for BQ access
pip install --user google-api-python-client oauth2client six==1.15.0
export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
# If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests
if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then
@ -67,7 +68,7 @@ then
time git clone --depth 1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master
# Needed for ios-binary-size
time pip install --user pyyaml pyjwt==1.7.1 pyOpenSSL cryptography requests
time pip install --user -r $DIR/requirements.macos.txt
# Store intermediate build files of ObjC tests into /tmpfs
# TODO(jtattermusch): this has likely been done to avoid running
@ -84,8 +85,8 @@ fi
if [ "${PREPARE_BUILD_INSTALL_DEPS_PYTHON}" == "true" ]
then
# python
time pip install --user virtualenv
time pip install --user --upgrade Mako tox setuptools==44.1.1 twisted pyyaml pyjwt==1.7.1 pyOpenSSL cryptography requests
time pip install --user -r $DIR/requirements.macos.txt
time pip install --user --upgrade virtualenv Mako tox setuptools==44.1.1 twisted
# Install Python 3.7 if it doesn't exist
if [ ! -f "/usr/local/bin/python3.7" ]; then

@ -0,0 +1,5 @@
cryptography==3.4.6
PyJWT==2.0.1
requests==2.25.1
scipy==1.5.4
tabulate==0.8.9

@ -0,0 +1,5 @@
cryptography==3.4.6
PyJWT==2.0.1
pyOpenSSL==20.0.1
PyYAML==5.4.1
requests==2.25.1

@ -20,7 +20,7 @@ cd $(dirname $0)/../../..
source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc
CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'`
CPUS=`python3 -c 'import multiprocessing; print(multiprocessing.cpu_count())'`
./tools/run_tests/start_port_server.py || true

@ -20,7 +20,7 @@ set -ex
# Enter the gRPC repo root
cd $(dirname $0)/../../..
AFFECTS_C_CPP=`python -c 'import os; \
AFFECTS_C_CPP=`python3 -c 'import os; \
import sys; \
sys.path.insert(0, "tools/run_tests/python_utils"); \
import filter_pull_request_tests as filter; \

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
#
# Copyright 2017 gRPC authors.
#

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
#
# Copyright 2017 gRPC authors.
#
@ -41,7 +41,7 @@ SANITIZE = {
}
if sys.argv[1] == '--schema':
print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns)
print(',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns))
sys.exit(0)
with open(sys.argv[1]) as f:

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
#
# Copyright 2017 gRPC authors.
#
@ -15,13 +15,13 @@
# limitations under the License.
""" Python utility to build opt and counters benchmarks """
import bm_constants
import argparse
import subprocess
import multiprocessing
import os
import shutil
import subprocess
import bm_constants
def _args():

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
#
# Copyright 2017 gRPC authors.
#

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
#
# Copyright 2017 gRPC authors.
#
@ -15,20 +15,19 @@
# limitations under the License.
""" Computes the diff between two bm runs and outputs significant results """
import bm_constants
import bm_speedup
import sys
import argparse
import collections
import json
import os
import subprocess
import sys
sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
import bm_json
import json
import bm_constants
import bm_json
import bm_speedup
import tabulate
import argparse
import collections
import subprocess
verbose = False
@ -38,9 +37,9 @@ def _median(ary):
ary = sorted(ary)
n = len(ary)
if n % 2 == 0:
return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0
return (ary[(n - 1) // 2] + ary[(n - 1) // 2 + 1]) / 2.0
else:
return ary[n / 2]
return ary[n // 2]
def _args():
@ -91,7 +90,7 @@ def _args():
def _maybe_print(str):
if verbose:
print str
print(str)
class Benchmark:
@ -136,14 +135,14 @@ def _read_json(filename, badjson_files, nonexistant_files):
with open(filename) as f:
r = f.read()
return json.loads(r)
except IOError, e:
except IOError as e:
if stripped in nonexistant_files:
nonexistant_files[stripped] += 1
else:
nonexistant_files[stripped] = 1
return None
except ValueError, e:
print r
except ValueError as e:
print(r)
if stripped in badjson_files:
badjson_files[stripped] += 1
else:
@ -166,6 +165,7 @@ def diff(bms, loops, regex, track, old, new, counters):
'bm_diff_%s/opt/%s' % (old, bm), '--benchmark_list_tests',
'--benchmark_filter=%s' % regex
]).splitlines():
line = line.decode('UTF-8')
stripped_line = line.strip().replace("/", "_").replace(
"<", "_").replace(">", "_").replace(", ", "_")
js_new_opt = _read_json(

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
#
# Copyright 2017 gRPC authors.
#
@ -15,26 +15,26 @@
# limitations under the License.
""" Runs the entire bm_*.py pipeline, and possible comments on the PR """
import bm_constants
import bm_build
import bm_run
import bm_diff
import sys
import os
import random
import argparse
import multiprocessing
import os
import random
import subprocess
import sys
sys.path.append(
os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests',
'python_utils'))
import check_on_pr
sys.path.append(
os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
'python_utils'))
import bm_build
import bm_constants
import bm_diff
import bm_run
import check_on_pr
import jobset

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
#
# Copyright 2017 gRPC authors.
#
@ -15,20 +15,20 @@
# limitations under the License.
""" Python utility to run opt and counters benchmarks and save json output """
import bm_constants
import argparse
import subprocess
import itertools
import multiprocessing
import os
import random
import itertools
import subprocess
import sys
import os
import bm_constants
import jobset
sys.path.append(
os.path.join(os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests',
'python_utils'))
import jobset
def _args():
@ -70,7 +70,8 @@ def _args():
args = argp.parse_args()
assert args.name
if args.loops < 3:
print "WARNING: This run will likely be noisy. Increase loops to at least 3."
print("WARNING: This run will likely be noisy. Increase loops to at "
"least 3.")
return args
@ -80,6 +81,7 @@ def _collect_bm_data(bm, cfg, name, regex, idx, loops):
'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_list_tests',
'--benchmark_filter=%s' % regex
]).splitlines():
line = line.decode('UTF-8')
stripped_line = line.strip().replace("/",
"_").replace("<", "_").replace(
">", "_").replace(", ", "_")

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python3
#
# Copyright 2017 gRPC authors.
#
@ -14,9 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from scipy import stats
import math
from scipy import stats
_DEFAULT_THRESHOLD = 1e-10
@ -63,5 +64,5 @@ def speedup(new, old, threshold=_DEFAULT_THRESHOLD):
if __name__ == "__main__":
new = [0.0, 0.0, 0.0, 0.0]
old = [2.96608e-06, 3.35076e-06, 3.45384e-06, 3.34407e-06]
print speedup(new, old, 1e-5)
print speedup(old, new, 1e-5)
print(speedup(new, old, 1e-5))
print(speedup(old, new, 1e-5))

@ -1,3 +1,4 @@
#!/usr/bin/env python3
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");

@ -55,7 +55,7 @@ def _access_token():
url='https://api.github.com/app/installations/%s/access_tokens'
% _INSTALLATION_ID,
headers={
'Authorization': 'Bearer %s' % _jwt_token().decode('ASCII'),
'Authorization': 'Bearer %s' % _jwt_token(),
'Accept': 'application/vnd.github.machine-man-preview+json',
})

@ -18,7 +18,7 @@ from __future__ import print_function
import re
import six
from subprocess import check_output
import subprocess
class TestSuite:
@ -126,10 +126,11 @@ def _get_changed_files(base_branch):
"""
# Get file changes between branch and merge-base of specified branch
# Not combined to be Windows friendly
base_commit = check_output(["git", "merge-base", base_branch,
"HEAD"]).rstrip()
return check_output(["git", "diff", base_commit, "--name-only",
"HEAD"]).splitlines()
base_commit = subprocess.check_output(
["git", "merge-base", base_branch, "HEAD"]).decode("UTF-8").rstrip()
return subprocess.check_output(
["git", "diff", base_commit, "--name-only",
"HEAD"]).decode("UTF-8").splitlines()
def _can_skip_tests(file_names, triggers):

@ -1,3 +1,4 @@
#!/usr/bin/env python3
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -96,6 +96,7 @@ def collect_latency(bm_name, args):
'bazel-bin/test/cpp/microbenchmarks/%s' % bm_name,
'--benchmark_list_tests'
]).splitlines():
line = line.decode('UTF-8')
link(line, '%s.txt' % fnize(line))
benchmarks.append(
jobset.JobSpec([
@ -150,6 +151,7 @@ def collect_perf(bm_name, args):
'bazel-bin/test/cpp/microbenchmarks/%s' % bm_name,
'--benchmark_list_tests'
]).splitlines():
line = line.decode('UTF-8')
link(line, '%s.svg' % fnize(line))
benchmarks.append(
jobset.JobSpec([

@ -1,5 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");

Loading…
Cancel
Save