Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Collect memory measurements of the minimal stack (#29164)

Browse Source 
* minstack measuring

* fix

* sort

* x

* x

* x

* x

* Automated change: Fix sanity tests

* review feedback

Co-authored-by: ctiller <ctiller@users.noreply.github.com>
pull/29226/head
Craig Tiller 3 years ago committed by GitHub
parent 6206700787
commit bf72db4180
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 66 additions and 34 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      test/core/memory_usage/client.cc
  2. 16
      test/core/memory_usage/memory_usage_test.cc
  3. 13
      test/core/memory_usage/server.cc
  4. 60
      tools/profiling/memory/memory_diff.py

11
test/core/memory_usage/client.cc
Unescape View File

@ -30,6 +30,7 @@
#include <grpc/support/log.h>
#include <grpc/support/time.h>
#include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/gpr/env.h"
#include "src/core/lib/gpr/string.h"
#include "src/core/lib/gpr/useful.h"
@ -214,6 +215,7 @@ std::pair<MemStats, MemStats> run_test_loop(int iterations, int* call_idx) {
ABSL_FLAG(std::string, target, "localhost:443", "Target host:port");
ABSL_FLAG(int, warmup, 100, "Warmup iterations");
ABSL_FLAG(int, benchmark, 1000, "Benchmark iterations");
ABSL_FLAG(bool, minstack, false, "Use minimal stack");
int main(int argc, char** argv) {
absl::ParseCommandLine(argc, argv);
@ -233,8 +235,15 @@ int main(int argc, char** argv) {
cq = grpc_completion_queue_create_for_next(nullptr);
std::vector<grpc_arg> args_vec;
if (absl::GetFlag(FLAGS_minstack)) {
args_vec.push_back(grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MINIMAL_STACK), 1));
}
grpc_channel_args args = {args_vec.size(), args_vec.data()};
channel = grpc_channel_create(absl::GetFlag(FLAGS_target).c_str(),
grpc_insecure_credentials_create(), nullptr);
grpc_insecure_credentials_create(), &args);
int call_idx = 0;
const int warmup_iterations = absl::GetFlag(FLAGS_warmup);

16
test/core/memory_usage/memory_usage_test.cc
Unescape View File

@ -34,6 +34,7 @@
ABSL_FLAG(int, warmup, 100, "Warmup iterations");
ABSL_FLAG(int, benchmark, 1000, "Benchmark iterations");
ABSL_FLAG(bool, minstack, false, "Use minimal stack");
class Subprocess {
public:
@ -74,15 +75,16 @@ int main(int argc, char** argv) {
/* start the server */
Subprocess svr({absl::StrCat(root, "/memory_usage_server",
gpr_subprocess_binary_extension()),
"--bind", grpc_core::JoinHostPort("::", port), "--nosecure"});
"--bind", grpc_core::JoinHostPort("::", port), "--nosecure",
absl::StrCat("--minstack=", absl::GetFlag(FLAGS_minstack))});
/* start the client */
Subprocess cli(
{absl::StrCat(root, "/memory_usage_client",
gpr_subprocess_binary_extension()),
"--target", grpc_core::JoinHostPort("127.0.0.1", port),
absl::StrCat("--warmup=", absl::GetFlag(FLAGS_warmup)),
absl::StrCat("--benchmark=", absl::GetFlag(FLAGS_benchmark))});
Subprocess cli({absl::StrCat(root, "/memory_usage_client",
gpr_subprocess_binary_extension()),
"--target", grpc_core::JoinHostPort("127.0.0.1", port),
absl::StrCat("--warmup=", absl::GetFlag(FLAGS_warmup)),
absl::StrCat("--benchmark=", absl::GetFlag(FLAGS_benchmark)),
absl::StrCat("--minstack=", absl::GetFlag(FLAGS_minstack))});
/* wait for completion */
if ((status = cli.Join()) != 0) {

13
test/core/memory_usage/server.cc
Unescape View File

@ -40,6 +40,7 @@
#include <grpc/support/log.h>
#include <grpc/support/time.h>
#include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/gprpp/host_port.h"
#include "test/core/end2end/data/ssl_test_data.h"
#include "test/core/memory_usage/memstats.h"
@ -154,6 +155,7 @@ static void sigint_handler(int /*x*/) { _exit(0); }
ABSL_FLAG(std::string, bind, "", "Bind host:port");
ABSL_FLAG(bool, secure, false, "Use security");
ABSL_FLAG(bool, minstack, false, "Use minimal stack");
int main(int argc, char** argv) {
absl::ParseCommandLine(argc, argv);
@ -180,17 +182,24 @@ int main(int argc, char** argv) {
cq = grpc_completion_queue_create_for_next(nullptr);
std::vector<grpc_arg> args_vec;
if (absl::GetFlag(FLAGS_minstack)) {
args_vec.push_back(grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MINIMAL_STACK), 1));
}
grpc_channel_args args = {args_vec.size(), args_vec.data()};
MemStats before_server_create = MemStats::Snapshot();
if (absl::GetFlag(FLAGS_secure)) {
grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {test_server1_key,
test_server1_cert};
grpc_server_credentials* ssl_creds = grpc_ssl_server_credentials_create(
nullptr, &pem_key_cert_pair, 1, 0, nullptr);
server = grpc_server_create(nullptr, nullptr);
server = grpc_server_create(&args, nullptr);
GPR_ASSERT(grpc_server_add_http2_port(server, addr.c_str(), ssl_creds));
grpc_server_credentials_release(ssl_creds);
} else {
server = grpc_server_create(nullptr, nullptr);
server = grpc_server_create(&args, nullptr);
GPR_ASSERT(grpc_server_add_http2_port(
server, addr.c_str(), grpc_insecure_server_credentials_create()));
}

60
tools/profiling/memory/memory_diff.py
Unescape View File

@ -49,6 +49,11 @@ _INTERESTING = {
(rb'server call memory usage: ([0-9\.]+) bytes per call', float),
}
_SCENARIOS = {
'default': [],
'minstack': ['--minstack'],
}
def _run():
"""Build with Bazel, then run, and extract interesting lines from the output."""
@ -56,24 +61,27 @@ def _run():
'tools/bazel', 'build', '-c', 'opt',
'test/core/memory_usage/memory_usage_test'
])
output = subprocess.check_output([
'bazel-bin/test/core/memory_usage/memory_usage_test',
'--warmup=10000',
'--benchmark=50000',
])
ret = {}
for line in output.splitlines():
for key, (pattern, conversion) in _INTERESTING.items():
m = re.match(pattern, line)
if m:
ret[key] = conversion(m.group(1))
for scenario, extra_args in _SCENARIOS.items():
try:
output = subprocess.check_output([
'bazel-bin/test/core/memory_usage/memory_usage_test',
'--warmup=10000',
'--benchmark=50000',
] + extra_args)
except subprocess.CalledProcessError as e:
print('Error running benchmark:', e)
continue
for line in output.splitlines():
for key, (pattern, conversion) in _INTERESTING.items():
m = re.match(pattern, line)
if m:
ret[scenario + ': ' + key] = conversion(m.group(1))
return ret
cur = _run()
new = None
print(cur)
old = None
if args.diff_base:
where_am_i = subprocess.check_output(
@ -81,24 +89,28 @@ if args.diff_base:
# checkout the diff base (="old")
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
new = _run()
old = _run()
finally:
# restore the original revision (="new")
# restore the original revision (="cur")
subprocess.check_call(['git', 'checkout', where_am_i])
text = ''
if new is None:
for key, value in cur.items():
if old is None:
print(cur)
for key, value in sorted(cur.items()):
text += '{}: {}\n'.format(key, value)
else:
print(cur, old)
diff_size = 0
for key, value in _INTERESTING.items():
if key in cur:
if key not in new:
text += '{}: {}\n'.format(key, value)
else:
diff_size += cur[key] - new[key]
text += '{}: {} -> {}\n'.format(key, cur[key], new[key])
for scenario in _SCENARIOS.keys():
for key, value in sorted(_INTERESTING.items()):
key = scenario + ': ' + key
if key in cur:
if key not in old:
text += '{}: {}\n'.format(key, cur[key])
else:
diff_size += cur[key] - old[key]
text += '{}: {} -> {}\n'.format(key, old[key], cur[key])
print("DIFF_SIZE: %f" % diff_size)
check_on_pr.label_increase_decrease_on_pr('per-call-memory', diff_size, 64)

Write Preview
Loading…
Cancel
Save