Chiebot-Mirror
/
grpc
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[build] Stop compiling every test.cc file for every config (#36197)

Browse Source 
Instead, build a library and re-use that across compilations.

This still invokes a link step per target, and we'll want to deal with that at some point too, but at least this makes some progress to not being as wasteful with our compilation resources.

Additionally: remove bm_pollset -- it was having some problems compiling, and we really don't need it anymore.

Closes #36197

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/36197 from ctiller:maintain-this-shite 9955026e23
PiperOrigin-RevId: 620946543
pull/36210/head
Craig Tiller 12 months ago committed by Copybara-Service
parent c079e533de
commit 26df942eb4
5 changed files with 14 additions and 275 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 15
      bazel/grpc_build_system.bzl
  2. 13
      test/cpp/microbenchmarks/BUILD
  3. 258
      test/cpp/microbenchmarks/bm_pollset.cc
  4. 2
      tools/internal_ci/linux/grpc_microbenchmark_diff_in_docker.sh
  5. 1
      tools/profiling/microbenchmarks/bm_diff/bm_constants.py

15
bazel/grpc_build_system.bzl
Unescape View File

@ -556,11 +556,22 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
**test_args
)
native.cc_library(
name = "%s_TEST_LIBRARY" % name,
testonly = 1,
srcs = srcs,
deps = core_deps,
tags = tags,
)
for poller_config in expand_tests(name, srcs, core_deps, tags, args, exclude_pollers, uses_polling, uses_event_engine, flaky):
if poller_config["srcs"] != srcs:
fail("srcs changed")
if poller_config["deps"] != core_deps:
fail("deps changed: %r --> %r" % (deps, poller_config["deps"]))
native.cc_test(
name = poller_config["name"],
srcs = poller_config["srcs"],
deps = poller_config["deps"],
deps = ["%s_TEST_LIBRARY" % name],
tags = poller_config["tags"],
args = poller_config["args"],
env = poller_config["env"],

13
test/cpp/microbenchmarks/BUILD
Unescape View File

@ -384,19 +384,6 @@ grpc_cc_test(
],
)
grpc_cc_test(
name = "bm_pollset",
srcs = ["bm_pollset.cc"],
args = grpc_benchmark_args(),
tags = [
"no_mac",
"no_windows",
],
deps = [
":helpers",
],
)
grpc_cc_library(
name = "bm_callback_test_service_impl",
testonly = 1,

258
test/cpp/microbenchmarks/bm_pollset.cc
Unescape View File

@ -1,258 +0,0 @@
//
//
// Copyright 2017 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
//
// Test out pollset latencies
#include <string.h>
#include <benchmark/benchmark.h>
#include <grpc/grpc.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include "src/core/lib/gpr/useful.h"
#include "src/core/lib/gprpp/crash.h"
#include "src/core/lib/gprpp/time.h"
#include "src/core/lib/iomgr/ev_posix.h"
#include "src/core/lib/iomgr/pollset.h"
#include "src/core/lib/iomgr/port.h"
#include "src/core/lib/iomgr/wakeup_fd_posix.h"
#include "test/core/util/test_config.h"
#include "test/cpp/microbenchmarks/helpers.h"
#include "test/cpp/util/test_config.h"
#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <unistd.h>
#endif
static void shutdown_ps(void* ps, grpc_error_handle /*error*/) {
grpc_pollset_destroy(static_cast<grpc_pollset*>(ps));
}
static void BM_CreateDestroyPollset(benchmark::State& state) {
size_t ps_sz = grpc_pollset_size();
grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_malloc(ps_sz));
gpr_mu* mu;
grpc_core::ExecCtx exec_ctx;
grpc_closure shutdown_ps_closure;
GRPC_CLOSURE_INIT(&shutdown_ps_closure, shutdown_ps, ps,
grpc_schedule_on_exec_ctx);
for (auto _ : state) {
memset(ps, 0, ps_sz);
grpc_pollset_init(ps, &mu);
gpr_mu_lock(mu);
grpc_pollset_shutdown(ps, &shutdown_ps_closure);
gpr_mu_unlock(mu);
grpc_core::ExecCtx::Get()->Flush();
}
grpc_core::ExecCtx::Get()->Flush();
gpr_free(ps);
}
BENCHMARK(BM_CreateDestroyPollset);
#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
static void BM_PollEmptyPollset_SpeedOfLight(benchmark::State& state) {
// equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage
// what the speed of light would be if we abstracted perfectly
int epfd = epoll_create1(0);
GPR_ASSERT(epfd != -1);
size_t nev = state.range(0);
size_t nfd = state.range(1);
epoll_event* ev = new epoll_event[nev];
std::vector<int> fds;
for (size_t i = 0; i < nfd; i++) {
fds.push_back(eventfd(0, 0));
epoll_event ev;
ev.events = EPOLLIN;
epoll_ctl(epfd, EPOLL_CTL_ADD, fds.back(), &ev);
}
for (auto _ : state) {
epoll_wait(epfd, ev, nev, 0);
}
for (auto fd : fds) {
close(fd);
}
close(epfd);
delete[] ev;
}
BENCHMARK(BM_PollEmptyPollset_SpeedOfLight)
->Args({1, 0})
->Args({1, 1})
->Args({1, 10})
->Args({1, 100})
->Args({1, 1000})
->Args({1, 10000})
->Args({1, 100000})
->Args({10, 1})
->Args({100, 1})
->Args({1000, 1});
#endif
static void BM_PollEmptyPollset(benchmark::State& state) {
size_t ps_sz = grpc_pollset_size();
grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz));
gpr_mu* mu;
grpc_pollset_init(ps, &mu);
grpc_core::ExecCtx exec_ctx;
gpr_mu_lock(mu);
for (auto _ : state) {
(void)grpc_pollset_work(ps, nullptr, grpc_core::Timestamp::ProcessEpoch());
}
grpc_closure shutdown_ps_closure;
GRPC_CLOSURE_INIT(&shutdown_ps_closure, shutdown_ps, ps,
grpc_schedule_on_exec_ctx);
grpc_pollset_shutdown(ps, &shutdown_ps_closure);
gpr_mu_unlock(mu);
grpc_core::ExecCtx::Get()->Flush();
gpr_free(ps);
}
BENCHMARK(BM_PollEmptyPollset);
static void BM_PollAddFd(benchmark::State& state) {
size_t ps_sz = grpc_pollset_size();
grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz));
gpr_mu* mu;
grpc_pollset_init(ps, &mu);
grpc_core::ExecCtx exec_ctx;
grpc_wakeup_fd wakeup_fd;
GPR_ASSERT(
GRPC_LOG_IF_ERROR("wakeup_fd_init", grpc_wakeup_fd_init(&wakeup_fd)));
grpc_fd* fd = grpc_fd_create(wakeup_fd.read_fd, "xxx", false);
for (auto _ : state) {
grpc_pollset_add_fd(ps, fd);
grpc_core::ExecCtx::Get()->Flush();
}
grpc_fd_orphan(fd, nullptr, nullptr, "xxx");
grpc_closure shutdown_ps_closure;
GRPC_CLOSURE_INIT(&shutdown_ps_closure, shutdown_ps, ps,
grpc_schedule_on_exec_ctx);
gpr_mu_lock(mu);
grpc_pollset_shutdown(ps, &shutdown_ps_closure);
gpr_mu_unlock(mu);
grpc_core::ExecCtx::Get()->Flush();
gpr_free(ps);
}
BENCHMARK(BM_PollAddFd);
class TestClosure : public grpc_closure {
public:
virtual ~TestClosure() {}
};
template <class F>
TestClosure* MakeTestClosure(F f) {
struct C : public TestClosure {
explicit C(F f) : f_(f) { GRPC_CLOSURE_INIT(this, C::cbfn, this, nullptr); }
static void cbfn(void* arg, grpc_error_handle /*error*/) {
C* p = static_cast<C*>(arg);
p->f_();
}
F f_;
};
return new C(f);
}
#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
static void BM_SingleThreadPollOneFd_SpeedOfLight(benchmark::State& state) {
// equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage
// what the speed of light would be if we abstracted perfectly
int epfd = epoll_create1(0);
GPR_ASSERT(epfd != -1);
epoll_event ev[100];
int fd = eventfd(0, EFD_NONBLOCK);
ev[0].events = EPOLLIN;
epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev[0]);
for (auto _ : state) {
int err;
do {
err = eventfd_write(fd, 1);
} while (err < 0 && errno == EINTR);
GPR_ASSERT(err == 0);
do {
err = epoll_wait(epfd, ev, GPR_ARRAY_SIZE(ev), 0);
} while (err < 0 && errno == EINTR);
GPR_ASSERT(err == 1);
eventfd_t value;
do {
err = eventfd_read(fd, &value);
} while (err < 0 && errno == EINTR);
GPR_ASSERT(err == 0);
}
close(fd);
close(epfd);
}
BENCHMARK(BM_SingleThreadPollOneFd_SpeedOfLight);
#endif
static void BM_SingleThreadPollOneFd(benchmark::State& state) {
size_t ps_sz = grpc_pollset_size();
grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz));
gpr_mu* mu;
grpc_pollset_init(ps, &mu);
grpc_core::ExecCtx exec_ctx;
grpc_wakeup_fd wakeup_fd;
(void)grpc_wakeup_fd_init(&wakeup_fd);
grpc_fd* wakeup = grpc_fd_create(wakeup_fd.read_fd, "wakeup_read", false);
grpc_pollset_add_fd(ps, wakeup);
bool done = false;
TestClosure* continue_closure = MakeTestClosure([&]() {
(void)grpc_wakeup_fd_consume_wakeup(&wakeup_fd);
if (!state.KeepRunning()) {
done = true;
return;
}
(void)grpc_wakeup_fd_wakeup(&wakeup_fd);
grpc_fd_notify_on_read(wakeup, continue_closure);
});
(void)grpc_wakeup_fd_wakeup(&wakeup_fd);
grpc_fd_notify_on_read(wakeup, continue_closure);
gpr_mu_lock(mu);
while (!done) {
(void)grpc_pollset_work(ps, nullptr, grpc_core::Timestamp::InfFuture());
}
grpc_fd_orphan(wakeup, nullptr, nullptr, "done");
wakeup_fd.read_fd = 0;
grpc_closure shutdown_ps_closure;
GRPC_CLOSURE_INIT(&shutdown_ps_closure, shutdown_ps, ps,
grpc_schedule_on_exec_ctx);
grpc_pollset_shutdown(ps, &shutdown_ps_closure);
gpr_mu_unlock(mu);
grpc_core::ExecCtx::Get()->Flush();
grpc_wakeup_fd_destroy(&wakeup_fd);
gpr_free(ps);
delete continue_closure;
}
BENCHMARK(BM_SingleThreadPollOneFd);
// Some distros have RunSpecifiedBenchmarks under the benchmark namespace,
// and others do not. This allows us to support both modes.
namespace benchmark {
void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); }
} // namespace benchmark
int main(int argc, char** argv) {
grpc::testing::TestEnvironment env(&argc, argv);
LibraryInitializer libInit;
::benchmark::Initialize(&argc, argv);
grpc::testing::InitTest(&argc, &argv, false);
benchmark::RunTheBenchmarksNamespaced();
return 0;
}

2
tools/internal_ci/linux/grpc_microbenchmark_diff_in_docker.sh
Unescape View File

@ -23,7 +23,7 @@ cd $(dirname $0)/../../..
time python3 -m pip install --user -r tools/internal_ci/helper_scripts/requirements.linux_perf.txt
# List of benchmarks that provide good signal for analyzing performance changes in pull requests
BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_chttp2_hpack bm_chttp2_transport bm_pollset"
BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_chttp2_hpack bm_chttp2_transport"
tools/run_tests/start_port_server.py

1
tools/profiling/microbenchmarks/bm_diff/bm_constants.py
Unescape View File

@ -23,7 +23,6 @@ _AVAILABLE_BENCHMARK_TESTS = [
"bm_cq",
"bm_chttp2_hpack",
"bm_chttp2_transport",
"bm_pollset",
]
_INTERESTING = (

Write Preview
Loading…
Cancel
Save