From d4be39a6ab08d7e2df44563e556704c01e753f6b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 15 Jun 2023 10:44:47 -0700 Subject: [PATCH] [fuzzing] Use a smaller max delay for writes than run-after (#33455) We want writes to participate in event re-ordering, but it's unlikely that we can sustain one byte per 500ms on all tests and keep them passing (which is the degenerate case right now). Tune write delays down to 50ms for the moment, though I expect we'll want to talk about going lower. --- ...zed-simple_request_fuzzer-5920082752503808 | 635 ++++++++++++++++++ test/core/end2end/end2end_test_fuzzer.cc | 1 + test/core/end2end/tests/simple_request.cc | 9 + .../fuzzing_event_engine.cc | 37 +- .../fuzzing_event_engine.h | 15 +- 5 files changed, 678 insertions(+), 19 deletions(-) create mode 100644 test/core/end2end/end2end_test_corpus/simple_request/clusterfuzz-testcase-minimized-simple_request_fuzzer-5920082752503808 diff --git a/test/core/end2end/end2end_test_corpus/simple_request/clusterfuzz-testcase-minimized-simple_request_fuzzer-5920082752503808 b/test/core/end2end/end2end_test_corpus/simple_request/clusterfuzz-testcase-minimized-simple_request_fuzzer-5920082752503808 new file mode 100644 index 00000000000..747e33b817a --- /dev/null +++ b/test/core/end2end/end2end_test_corpus/simple_request/clusterfuzz-testcase-minimized-simple_request_fuzzer-5920082752503808 @@ -0,0 +1,635 @@ +test_id: 157 +event_engine_actions { + run_delay: 0 + run_delay: 704643072 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 72057594037927936 + run_delay: 1 + run_delay: 10944512 + run_delay: 123645471490048 + run_delay: 0 + run_delay: 121382057085440 + run_delay: 495766732800 + assign_ports: 46 + connections { + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 44 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 41728 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 256 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 41728 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 41728 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + write_size: 0 + } +} diff --git a/test/core/end2end/end2end_test_fuzzer.cc b/test/core/end2end/end2end_test_fuzzer.cc index f729fc09b87..0a91102d4f7 100644 --- a/test/core/end2end/end2end_test_fuzzer.cc +++ b/test/core/end2end/end2end_test_fuzzer.cc @@ -122,6 +122,7 @@ DEFINE_PROTO_FUZZER(const core_end2end_test_fuzzer::Msg& msg) { [actions = msg.event_engine_actions()]() { FuzzingEventEngine::Options options; options.max_delay_run_after = std::chrono::milliseconds(500); + options.max_delay_write = std::chrono::milliseconds(50); return std::make_unique(options, actions); }); auto engine = diff --git a/test/core/end2end/tests/simple_request.cc b/test/core/end2end/tests/simple_request.cc index ceeb7699562..328b5bfe048 100644 --- a/test/core/end2end/tests/simple_request.cc +++ b/test/core/end2end/tests/simple_request.cc @@ -23,6 +23,7 @@ #include #include "absl/strings/match.h" +#include "absl/strings/str_cat.h" #include "absl/types/optional.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -32,6 +33,7 @@ #include "src/core/lib/debug/stats.h" #include "src/core/lib/debug/stats_data.h" +#include "src/core/lib/gprpp/crash.h" #include "src/core/lib/gprpp/time.h" #include "test/core/end2end/end2end_tests.h" @@ -60,6 +62,13 @@ void SimpleRequestBody(CoreEnd2endTest& test) { .RecvStatusOnClient(server_status); auto s = test.RequestCall(101); test.Expect(101, true); + test.Expect( + 1, CoreEnd2endTest::MaybePerformAction{[&](bool success) { + Crash(absl::StrCat( + "Unexpected completion of client side call: success=", + success ? "true" : "false", " status=", server_status.ToString(), + " initial_md=", server_initial_metadata.ToString())); + }}); test.Step(); EXPECT_NE(s.GetPeer(), absl::nullopt); CheckPeer(*s.GetPeer()); diff --git a/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc b/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc index aaaf2322ce9..9779e0f0440 100644 --- a/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc +++ b/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc @@ -70,7 +70,7 @@ gpr_timespec (*g_orig_gpr_now_impl)(gpr_clock_type clock_type); FuzzingEventEngine::FuzzingEventEngine( Options options, const fuzzing_event_engine::Actions& actions) - : max_delay_run_after_(options.max_delay_run_after) { + : max_delay_{options.max_delay_write, options.max_delay_run_after} { tasks_by_id_.clear(); tasks_by_time_.clear(); next_task_id_ = 1; @@ -107,8 +107,7 @@ FuzzingEventEngine::FuzzingEventEngine( for (const auto& delay_ns : actions.run_delay()) { Duration delay = std::chrono::nanoseconds(delay_ns); - task_delays_.push( - grpc_core::Clamp(delay, Duration(0), max_delay_run_after_)); + task_delays_.push(delay); } previous_pick_port_functions_ = grpc_set_pick_port_functions( @@ -295,6 +294,7 @@ bool FuzzingEventEngine::EndpointMiddle::Write(SliceBuffer* data, int index) { Slice::FromCopiedBuffer(pending[index])); pending[index].clear(); g_fuzzing_event_engine->RunLocked( + RunType::kWrite, [cb = std::move(pending_read[peer_index]->on_read)]() mutable { cb(absl::OkStatus()); }); @@ -320,12 +320,13 @@ void FuzzingEventEngine::FuzzingEndpoint::ScheduleDelayedWrite( std::shared_ptr middle, int index, absl::AnyInvocable on_writable, SliceBuffer* data) { g_fuzzing_event_engine->RunLocked( - [middle = std::move(middle), index, data, - on_writable = std::move(on_writable)]() mutable { + RunType::kWrite, [middle = std::move(middle), index, data, + on_writable = std::move(on_writable)]() mutable { grpc_core::ReleasableMutexLock lock(&*mu_); GPR_ASSERT(middle->writing[index]); if (middle->closed[index]) { g_fuzzing_event_engine->RunLocked( + RunType::kRunAfter, [on_writable = std::move(on_writable)]() mutable { on_writable(absl::InternalError("Endpoint closed")); }); @@ -347,6 +348,7 @@ FuzzingEventEngine::FuzzingEndpoint::~FuzzingEndpoint() { middle_->closed[my_index()] = true; if (middle_->pending_read[my_index()].has_value()) { g_fuzzing_event_engine->RunLocked( + RunType::kRunAfter, [cb = std::move(middle_->pending_read[my_index()]->on_read)]() mutable { cb(absl::InternalError("Endpoint closed")); }); @@ -355,6 +357,7 @@ FuzzingEventEngine::FuzzingEndpoint::~FuzzingEndpoint() { if (!middle_->writing[peer_index()] && middle_->pending_read[peer_index()].has_value()) { g_fuzzing_event_engine->RunLocked( + RunType::kRunAfter, [cb = std::move( middle_->pending_read[peer_index()]->on_read)]() mutable { cb(absl::InternalError("Endpoint closed")); @@ -373,7 +376,7 @@ bool FuzzingEventEngine::FuzzingEndpoint::Read( // If the endpoint is closed, fail asynchronously. if (middle_->closed[peer_index()]) { g_fuzzing_event_engine->RunLocked( - [on_read = std::move(on_read)]() mutable { + RunType::kRunAfter, [on_read = std::move(on_read)]() mutable { on_read(absl::InternalError("Endpoint closed")); }); return false; @@ -427,14 +430,15 @@ EventEngine::ConnectionHandle FuzzingEventEngine::Connect( listener_port, g_fuzzing_event_engine->AllocatePort()); auto ep1 = std::make_unique(middle, 0); auto ep2 = std::make_unique(middle, 1); - RunLocked([listener, ep1 = std::move(ep1)]() mutable { + RunLocked(RunType::kRunAfter, [listener, + ep1 = std::move(ep1)]() mutable { listener->on_accept( std::move(ep1), listener->memory_allocator_factory->CreateMemoryAllocator( "fuzzing")); }); - RunLocked([on_connect = std::move(on_connect), - ep2 = std::move(ep2)]() mutable { + RunLocked(RunType::kRunAfter, [on_connect = std::move(on_connect), + ep2 = std::move(ep2)]() mutable { on_connect(std::move(ep2)); }); return; @@ -442,9 +446,10 @@ EventEngine::ConnectionHandle FuzzingEventEngine::Connect( } } // Fail: no such listener. - RunLocked([on_connect = std::move(on_connect)]() mutable { - on_connect(absl::InvalidArgumentError("No listener found")); - }); + RunLocked(RunType::kRunAfter, + [on_connect = std::move(on_connect)]() mutable { + on_connect(absl::InvalidArgumentError("No listener found")); + }); }); return ConnectionHandle{{task_handle.keys[0], task_handle.keys[1]}}; } @@ -478,15 +483,17 @@ EventEngine::TaskHandle FuzzingEventEngine::RunAfter( Duration when, absl::AnyInvocable closure) { grpc_core::MutexLock lock(&*mu_); // (b/258949216): Cap it to one year to avoid integer overflow errors. - return RunAfterLocked(std::min(when, kOneYear), std::move(closure)); + return RunAfterLocked(RunType::kRunAfter, std::min(when, kOneYear), + std::move(closure)); } EventEngine::TaskHandle FuzzingEventEngine::RunAfterLocked( - Duration when, absl::AnyInvocable closure) { + RunType run_type, Duration when, absl::AnyInvocable closure) { const intptr_t id = next_task_id_; ++next_task_id_; if (!task_delays_.empty()) { - when += task_delays_.front(); + when += grpc_core::Clamp(task_delays_.front(), Duration::zero(), + max_delay_[static_cast(run_type)]); task_delays_.pop(); } auto task = std::make_shared(id, std::move(closure)); diff --git a/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h b/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h index fe6430abb85..870505bc873 100644 --- a/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h +++ b/test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h @@ -53,6 +53,7 @@ class FuzzingEventEngine : public EventEngine { public: struct Options { Duration max_delay_run_after = std::chrono::seconds(30); + Duration max_delay_write = std::chrono::seconds(30); }; explicit FuzzingEventEngine(Options options, const fuzzing_event_engine::Actions& actions); @@ -106,6 +107,11 @@ class FuzzingEventEngine : public EventEngine { void UnsetGlobalHooks() ABSL_LOCKS_EXCLUDED(mu_); private: + enum class RunType { + kWrite, + kRunAfter, + }; + // One pending task to be run. struct Task { Task(intptr_t id, absl::AnyInvocable closure) @@ -225,12 +231,13 @@ class FuzzingEventEngine : public EventEngine { const int index_; }; - void RunLocked(absl::AnyInvocable closure) + void RunLocked(RunType run_type, absl::AnyInvocable closure) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { - RunAfterLocked(Duration::zero(), std::move(closure)); + RunAfterLocked(run_type, Duration::zero(), std::move(closure)); } - TaskHandle RunAfterLocked(Duration when, absl::AnyInvocable closure) + TaskHandle RunAfterLocked(RunType run_type, Duration when, + absl::AnyInvocable closure) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Allocate a port. Considered fuzzer selected port orderings first, and then @@ -254,7 +261,7 @@ class FuzzingEventEngine : public EventEngine { Duration exponential_gate_time_increment_ ABSL_GUARDED_BY(mu_) = std::chrono::milliseconds(1); - const Duration max_delay_run_after_; + const Duration max_delay_[2]; intptr_t next_task_id_ ABSL_GUARDED_BY(mu_); intptr_t current_tick_ ABSL_GUARDED_BY(now_mu_); Time now_ ABSL_GUARDED_BY(now_mu_);