Add per-call memory bloat report to CI (#29037)

* Add per-call memory bloat report to CI

* fix

* test

* Automated change: Fix sanity tests

* Update grpc_bloat_diff_in_docker.sh

* Update memory_diff.py

* review feedback

* prep for new job

Co-authored-by: ctiller <ctiller@users.noreply.github.com>

tools/internal_ci/linux/grpc_memory_diff.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian11_x64
+export DOCKER_RUN_SCRIPT=tools/internal_ci/linux/grpc_memory_diff_in_docker.sh
+# The check_on_pr.py needs access to the key to post status on github PRs,
+# so we mount the keystore dir to the docker container.
+export EXTRA_DOCKER_ARGS="-v ${KOKORO_KEYSTORE_DIR}:/kokoro_keystore -e KOKORO_KEYSTORE_DIR=/kokoro_keystore"
+exec tools/run_tests/dockerize/build_and_run_docker.sh

tools/internal_ci/linux/grpc_memory_diff_in_docker.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright 2022 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+# some extra pip packages are needed for the check_on_pr.py script to work
+# TODO(jtattermusch): avoid needing to install these pip packages each time
+time python3 -m pip install --user -r tools/internal_ci/helper_scripts/requirements.linux_perf.txt
+
+tools/run_tests/start_port_server.py
+
+tools/internal_ci/linux/run_if_c_cpp_modified.sh tools/profiling/memory/memory_diff.py \
+  -d "origin/$KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH"

tools/profiling/memory/memory_diff.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+#
+# Copyright 2022 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import csv
+import glob
+import math
+import multiprocessing
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+
+sys.path.append(
+    os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests',
+                 'python_utils'))
+import check_on_pr
+
+argp = argparse.ArgumentParser(description='Perform diff on memory benchmarks')
+
+argp.add_argument('-d',
+                  '--diff_base',
+                  type=str,
+                  help='Commit or branch to compare the current one to')
+
+argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count())
+
+args = argp.parse_args()
+
+_INTERESTING = {
+    'client call':
+        (rb'client call memory usage: ([0-9\.]+) bytes per call', float),
+    'server call':
+        (rb'server call memory usage: ([0-9\.]+) bytes per call', float),
+}
+
+
+def _run():
+    """Build with Bazel, then run, and extract interesting lines from the output."""
+    subprocess.check_call([
+        'tools/bazel', 'build', '-c', 'opt',
+        'test/core/memory_usage/memory_usage_test'
+    ])
+    output = subprocess.check_output([
+        'bazel-bin/test/core/memory_usage/memory_usage_test',
+        '--warmup=10000',
+        '--benchmark=50000',
+    ])
+    ret = {}
+    for line in output.splitlines():
+        for key, (pattern, conversion) in _INTERESTING.items():
+            m = re.match(pattern, line)
+            if m:
+                ret[key] = conversion(m.group(1))
+    return ret
+
+
+cur = _run()
+new = None
+
+print(cur)
+
+if args.diff_base:
+    where_am_i = subprocess.check_output(
+        ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode().strip()
+    # checkout the diff base (="old")
+    subprocess.check_call(['git', 'checkout', args.diff_base])
+    try:
+        new = _run()
+    finally:
+        # restore the original revision (="new")
+        subprocess.check_call(['git', 'checkout', where_am_i])
+
+text = ''
+if new is None:
+    for key, value in cur.items():
+        text += '{}: {}\n'.format(key, value)
+else:
+    diff_size = 0
+    for key, value in _INTERESTING.items():
+        if key in cur:
+            if key not in new:
+                text += '{}: {}\n'.format(key, value)
+            else:
+                diff_size += cur[key] - new[key]
+                text += '{}: {} -> {}\n'.format(key, cur[key], new[key])
+
+    print("DIFF_SIZE: %f" % diff_size)
+    check_on_pr.label_increase_decrease_on_pr('per-call-memory', diff_size, 64)
+
+print(text)
+check_on_pr.check_on_pr('Memory Difference', '```\n%s\n```' % text)

tools/run_tests/python_utils/check_on_pr.py

@@ -41,6 +41,12 @@ _CHANGE_LABELS = {
     3: 'high',
 }
 
+_INCREASE_DECREASE = {
+    -1: 'decrease',
+    0: 'neutral',
+    1: 'increase',
+}
+
 
 def _jwt_token():
     github_app_key = open(
@@ -155,7 +161,7 @@ def check_on_pr(name, summary, success=True):
           json.dumps(resp.json(), indent=2))
 
 
-def label_significance_on_pr(name, change):
+def label_significance_on_pr(name, change, labels=_CHANGE_LABELS):
     """Add a label to the PR indicating the significance of the check.
 
     Requires environment variable 'KOKORO_GITHUB_PULL_REQUEST_NUMBER' to indicate which pull request
@@ -165,11 +171,11 @@ def label_significance_on_pr(name, change):
       name: The name of the label.
       value: A str in Markdown to be used as the detail information of the label.
     """
-    if change < min(list(_CHANGE_LABELS.keys())):
+    if change < min(list(labels.keys())):
-        change = min(list(_CHANGE_LABELS.keys()))
+        change = min(list(labels.keys()))
-    if change > max(list(_CHANGE_LABELS.keys())):
+    if change > max(list(labels.keys())):
-        change = max(list(_CHANGE_LABELS.keys()))
+        change = max(list(labels.keys()))
-    value = _CHANGE_LABELS[change]
+    value = labels[change]
     if 'KOKORO_GIT_COMMIT' not in os.environ:
         print('Missing KOKORO_GIT_COMMIT env var: not checking')
         return
@@ -192,3 +198,12 @@ def label_significance_on_pr(name, change):
         method='PUT',
         json=new)
     print('Result of setting labels on PR:', resp.text)
+
+
+def label_increase_decrease_on_pr(name, change, significant):
+    if change <= -significant:
+        label_significance_on_pr(name, -1, _INCREASE_DECREASE)
+    elif change >= significant:
+        label_significance_on_pr(name, 1, _INCREASE_DECREASE)
+    else:
+        label_significance_on_pr(name, 0, _INCREASE_DECREASE)