[fuzzing] Implement endpoints for FuzzingEventEngine (#32689)

Implement listeners, connection, endpoints for `FuzzingEventEngine`.
Allows the fuzzer to select write sizes and delays, connection delays,
and port assignments.

I made a few modifications to the test suite to admit this event engine
to pass the client & server tests:
1. the test factories return shared_ptr<> to admit us to return the same
event engine for both the oracle and the implementation - necessary
because FuzzingEventEngine forms a closed world of addresses & ports.
2. removed the WaitForSingleOwner calls - these seem unnecessary, and we
don't ask our users to do this - tested existing linux tests 1000x
across debug, asan, tsan with this change

Additionally, the event engine overrides the global port picker logic so
that port assignments are made by the fuzzer too.

This PR is a step along a longer journey, and has some outstanding
brethren PR's, and some follow-up work:
* #32603 will convert all the core e2e tests into a more malleable form
* we'll then use #32667 to turn all of these into fuzzers
* finally we'll integrate this into that work and turn all core e2e
tests into fuzzers over timer & callback reorderings and io
size/spacings

---------

Co-authored-by: ctiller <ctiller@users.noreply.github.com>

CMakeLists.txt

@@ -11702,6 +11702,8 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_POSIX)
     test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc
     test/core/event_engine/test_suite/event_engine_test_framework.cc
     test/core/event_engine/test_suite/fuzzing_event_engine_test.cc
+    test/core/event_engine/test_suite/tests/client_test.cc
+    test/core/event_engine/test_suite/tests/server_test.cc
     test/core/event_engine/test_suite/tests/timer_test.cc
     third_party/googletest/googletest/src/gtest-all.cc
     third_party/googletest/googlemock/src/gmock-all.cc

build_autogenerated.yaml

@@ -7902,6 +7902,8 @@ targets:
   - test/core/event_engine/event_engine_test_utils.h
   - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h
   - test/core/event_engine/test_suite/event_engine_test_framework.h
+  - test/core/event_engine/test_suite/tests/client_test.h
+  - test/core/event_engine/test_suite/tests/server_test.h
   - test/core/event_engine/test_suite/tests/timer_test.h
   src:
   - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.proto
@@ -7909,6 +7911,8 @@ targets:
   - test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc
   - test/core/event_engine/test_suite/event_engine_test_framework.cc
   - test/core/event_engine/test_suite/fuzzing_event_engine_test.cc
+  - test/core/event_engine/test_suite/tests/client_test.cc
+  - test/core/event_engine/test_suite/tests/server_test.cc
   - test/core/event_engine/test_suite/tests/timer_test.cc
   deps:
   - grpc_unsecure

test/core/end2end/fuzzers/api_fuzzer.cc

@@ -812,7 +812,6 @@ DEFINE_PROTO_FUZZER(const api_fuzzer::Msg& msg) {
       });
   auto engine =
       std::dynamic_pointer_cast<FuzzingEventEngine>(GetDefaultEventEngine());
-  FuzzingEventEngine::SetGlobalNowImplEngine(engine.get());
   grpc_init();
   grpc_set_tcp_client_impl(&fuzz_tcp_client_vtable);
   grpc_timer_manager_set_threading(false);
@@ -1208,5 +1207,5 @@ DEFINE_PROTO_FUZZER(const api_fuzzer::Msg& msg) {
 
   grpc_resource_quota_unref(g_resource_quota);
   grpc_shutdown_blocking();
-  FuzzingEventEngine::UnsetGlobalNowImplEngine(engine.get());
+  engine->UnsetGlobalHooks();
 }

test/core/event_engine/event_engine_test_utils.cc

@@ -131,6 +131,7 @@ absl::Status SendValidatePayload(absl::string_view data,
     read_slice_buf.MoveFirstNBytesIntoSliceBuffer(read_slice_buf.Length(),
                                                   read_store_buf);
     if (receive_endpoint->Read(read_cb, &read_slice_buf, &args)) {
+      GPR_ASSERT(read_slice_buf.Length() != 0);
       read_cb(absl::OkStatus());
     }
   };

test/core/event_engine/fuzzing_event_engine/BUILD

@@ -30,6 +30,7 @@ grpc_cc_library(
         "//:event_engine_base_hdrs",
         "//src/core:default_event_engine",
         "//src/core:time",
+        "//test/core/util:grpc_test_util",
     ],
 )
 

test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.cc

@@ -18,20 +18,44 @@
 
 #include <algorithm>
 #include <chrono>
+#include <limits>
 #include <ratio>
+#include <type_traits>
 #include <vector>
 
+#include "absl/memory/memory.h"
+#include "absl/strings/str_cat.h"
+
+#include <grpc/event_engine/slice.h>
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/event_engine/tcp_socket_utils.h"
 #include "src/core/lib/gprpp/time.h"
 #include "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h"
+#include "test/core/util/port.h"
+
+// IWYU pragma: no_include <sys/socket.h>
 
 extern gpr_timespec (*gpr_now_impl)(gpr_clock_type clock_type);
 
 namespace grpc_event_engine {
 namespace experimental {
 
+namespace {
+
+// Inside the fuzzing event engine we consider everything is bound to a single
+// loopback device. It cannot reach any other devices, and shares all ports
+// between ipv4 and ipv6.
+
+EventEngine::ResolvedAddress PortToAddress(int port) {
+  return URIToResolvedAddress(absl::StrCat("ipv4:127.0.0.1:", port)).value();
+}
+
+}  // namespace
+
+grpc_core::NoDestruct<grpc_core::Mutex> FuzzingEventEngine::mu_;
+
 namespace {
 const intptr_t kTaskHandleSalt = 12345;
 FuzzingEventEngine* g_fuzzing_event_engine = nullptr;
@@ -53,6 +77,22 @@ FuzzingEventEngine::FuzzingEventEngine(
   // epoch to allow for some fancy atomic stuff.
   now_ = Time() + std::chrono::seconds(5);
 
+  // Allow the fuzzer to assign ports.
+  // Once this list is exhausted, we fall back to a deterministic algorithm.
+  for (auto port : actions.assign_ports()) {
+    free_ports_.push(port);
+    fuzzer_mentioned_ports_.insert(port);
+  }
+
+  // Fill the write sizes queue for future connections.
+  for (const auto& connection : actions.connections()) {
+    std::queue<size_t> write_sizes;
+    for (auto size : connection.write_size()) {
+      write_sizes.push(size);
+    }
+    write_sizes_for_future_connections_.emplace(std::move(write_sizes));
+  }
+
   // Whilst a fuzzing EventEngine is active we override grpc's now function.
   grpc_core::TestOnlySetProcessEpoch(NowAsTimespec(GPR_CLOCK_MONOTONIC));
 
@@ -78,10 +118,21 @@ FuzzingEventEngine::FuzzingEventEngine(
   for (const auto& delay : actions.run_delay()) {
     update_delay(&task_delays_, delay, std::chrono::seconds(30));
   }
+
+  GPR_ASSERT(g_fuzzing_event_engine == nullptr);
+  g_fuzzing_event_engine = this;
+  g_orig_gpr_now_impl = gpr_now_impl;
+  gpr_now_impl = GlobalNowImpl;
+  previous_pick_port_functions_ = grpc_set_pick_port_functions(
+      grpc_pick_port_functions{+[]() -> int {
+                                 grpc_core::MutexLock lock(&*mu_);
+                                 return g_fuzzing_event_engine->AllocatePort();
+                               },
+                               +[](int) {}});
 }
 
 void FuzzingEventEngine::FuzzingDone() {
-  grpc_core::MutexLock lock(&mu_);
+  grpc_core::MutexLock lock(&*mu_);
   tick_increments_.clear();
 }
 
@@ -97,7 +148,7 @@ gpr_timespec FuzzingEventEngine::NowAsTimespec(gpr_clock_type clock_type) {
 void FuzzingEventEngine::Tick() {
   std::vector<absl::AnyInvocable<void()>> to_run;
   {
-    grpc_core::MutexLock lock(&mu_);
+    grpc_core::MutexLock lock(&*mu_);
     // Increment time
     auto tick_it = tick_increments_.find(current_tick_);
     if (tick_it != tick_increments_.end()) {
@@ -125,25 +176,270 @@ void FuzzingEventEngine::Tick() {
 }
 
 FuzzingEventEngine::Time FuzzingEventEngine::Now() {
-  grpc_core::MutexLock lock(&mu_);
+  grpc_core::MutexLock lock(&*mu_);
   return now_;
 }
 
+int FuzzingEventEngine::AllocatePort() {
+  // If the fuzzer selected some port orderings, do that first.
+  if (!free_ports_.empty()) {
+    int p = free_ports_.front();
+    free_ports_.pop();
+    return p;
+  }
+  // Otherwise just scan through starting at one and skipping any ports
+  // that were in the fuzzers initial list.
+  while (true) {
+    int p = next_free_port_++;
+    if (fuzzer_mentioned_ports_.count(p) == 0) {
+      return p;
+    }
+  }
+}
+
 absl::StatusOr<std::unique_ptr<EventEngine::Listener>>
-FuzzingEventEngine::CreateListener(Listener::AcceptCallback,
-                                   absl::AnyInvocable<void(absl::Status)>,
-                                   const EndpointConfig&,
-                                   std::unique_ptr<MemoryAllocatorFactory>) {
-  abort();
+FuzzingEventEngine::CreateListener(
+    Listener::AcceptCallback on_accept,
+    absl::AnyInvocable<void(absl::Status)> on_shutdown, const EndpointConfig&,
+    std::unique_ptr<MemoryAllocatorFactory> memory_allocator_factory) {
+  grpc_core::MutexLock lock(&*mu_);
+  // Create a listener and register it into the set of listener info in the
+  // event engine.
+  return absl::make_unique<FuzzingListener>(
+      *listeners_
+           .emplace(std::make_shared<ListenerInfo>(
+               std::move(on_accept), std::move(on_shutdown),
+               std::move(memory_allocator_factory)))
+           .first);
+}
+
+FuzzingEventEngine::FuzzingListener::~FuzzingListener() {
+  grpc_core::MutexLock lock(&*mu_);
+  g_fuzzing_event_engine->listeners_.erase(info_);
+}
+
+bool FuzzingEventEngine::IsPortUsed(int port) {
+  // Return true if a port is bound to a listener.
+  for (const auto& listener : listeners_) {
+    if (std::find(listener->ports.begin(), listener->ports.end(), port) !=
+        listener->ports.end()) {
+      return true;
+    }
+  }
+  return false;
+}
+
+absl::StatusOr<int> FuzzingEventEngine::FuzzingListener::Bind(
+    const ResolvedAddress& addr) {
+  // Extract the port from the address (or fail if non-localhost).
+  auto port = ResolvedAddressGetPort(addr);
+  grpc_core::MutexLock lock(&*mu_);
+  // Check that the listener hasn't already been started.
+  if (info_->started) return absl::InternalError("Already started");
+  if (port != 0) {
+    // If the port is non-zero, check that it's not already in use.
+    if (g_fuzzing_event_engine->IsPortUsed(port)) {
+      return absl::InternalError("Port in use");
+    }
+  } else {
+    // If the port is zero, allocate a new one.
+    do {
+      port = g_fuzzing_event_engine->AllocatePort();
+    } while (g_fuzzing_event_engine->IsPortUsed(port));
+  }
+  // Add the port to the listener.
+  info_->ports.push_back(port);
+  return port;
+}
+
+absl::Status FuzzingEventEngine::FuzzingListener::Start() {
+  // Start the listener or fail if it's already started.
+  grpc_core::MutexLock lock(&*mu_);
+  if (info_->started) return absl::InternalError("Already started");
+  info_->started = true;
+  return absl::OkStatus();
 }
 
+bool FuzzingEventEngine::EndpointMiddle::Write(SliceBuffer* data, int index) {
+  GPR_ASSERT(!closed);
+  const int peer_index = 1 - index;
+  if (data->Length() == 0) return true;
+  size_t write_len = std::numeric_limits<size_t>::max();
+  // Check the write_sizes queue for fuzzer imposed restrictions on this write
+  // size. This allows the fuzzer to force small writes to be seen by the
+  // reader.
+  if (!write_sizes[index].empty()) {
+    write_len = write_sizes[index].front();
+    write_sizes[index].pop();
+  }
+  if (write_len > data->Length()) {
+    write_len = data->Length();
+  }
+  // If the write_len is zero, we still need to write something, so we write one
+  // byte.
+  if (write_len == 0) write_len = 1;
+  // Expand the pending buffer.
+  size_t prev_len = pending[index].size();
+  pending[index].resize(prev_len + write_len);
+  // Move bytes from the to-write data into the pending buffer.
+  data->MoveFirstNBytesIntoBuffer(write_len, pending[index].data() + prev_len);
+  // If there was a pending read, then we can fulfill it.
+  if (pending_read[peer_index].has_value()) {
+    pending_read[peer_index]->buffer->Append(
+        Slice::FromCopiedBuffer(pending[index]));
+    pending[index].clear();
+    g_fuzzing_event_engine->RunLocked(
+        [cb = std::move(pending_read[peer_index]->on_read)]() mutable {
+          cb(absl::OkStatus());
+        });
+    pending_read[peer_index].reset();
+  }
+  return data->Length() == 0;
+}
+
+bool FuzzingEventEngine::FuzzingEndpoint::Write(
+    absl::AnyInvocable<void(absl::Status)> on_writable, SliceBuffer* data,
+    const WriteArgs*) {
+  grpc_core::MutexLock lock(&*mu_);
+  // If the endpoint is closed, then we fail the write.
+  if (middle_->closed) {
+    g_fuzzing_event_engine->RunLocked(
+        [on_writable = std::move(on_writable)]() mutable {
+          on_writable(absl::InternalError("Endpoint closed"));
+        });
+    return false;
+  }
+  // If the write succeeds immediately, then we return true.
+  if (middle_->Write(data, my_index())) return true;
+  ScheduleDelayedWrite(middle_, my_index(), std::move(on_writable), data);
+  return false;
+}
+
+void FuzzingEventEngine::FuzzingEndpoint::ScheduleDelayedWrite(
+    std::shared_ptr<EndpointMiddle> middle, int index,
+    absl::AnyInvocable<void(absl::Status)> on_writable, SliceBuffer* data) {
+  g_fuzzing_event_engine->RunLocked(
+      [middle = std::move(middle), index, data,
+       on_writable = std::move(on_writable)]() mutable {
+        grpc_core::MutexLock lock(&*mu_);
+        if (middle->closed) {
+          g_fuzzing_event_engine->RunLocked(
+              [on_writable = std::move(on_writable)]() mutable {
+                on_writable(absl::InternalError("Endpoint closed"));
+              });
+          return;
+        }
+        if (middle->Write(data, index)) {
+          on_writable(absl::OkStatus());
+          return;
+        }
+        ScheduleDelayedWrite(std::move(middle), index, std::move(on_writable),
+                             data);
+      });
+}
+
+FuzzingEventEngine::FuzzingEndpoint::~FuzzingEndpoint() {
+  grpc_core::MutexLock lock(&*mu_);
+  middle_->closed = true;
+  for (int i = 0; i < 2; i++) {
+    if (middle_->pending_read[i].has_value()) {
+      g_fuzzing_event_engine->RunLocked(
+          [cb = std::move(middle_->pending_read[i]->on_read)]() mutable {
+            cb(absl::InternalError("Endpoint closed"));
+          });
+      middle_->pending_read[i].reset();
+    }
+  }
+}
+
+bool FuzzingEventEngine::FuzzingEndpoint::Read(
+    absl::AnyInvocable<void(absl::Status)> on_read, SliceBuffer* buffer,
+    const ReadArgs*) {
+  buffer->Clear();
+  grpc_core::MutexLock lock(&*mu_);
+  // If the endpoint is closed, fail asynchronously.
+  if (middle_->closed) {
+    g_fuzzing_event_engine->RunLocked([on_read = std::move(on_read)]() mutable {
+      on_read(absl::InternalError("Endpoint closed"));
+    });
+    return false;
+  }
+  if (middle_->pending[peer_index()].empty()) {
+    // If the endpoint has no pending data, then we need to wait for a write.
+    middle_->pending_read[my_index()] = PendingRead{std::move(on_read), buffer};
+    return false;
+  } else {
+    // If the endpoint has pending data, then we can fulfill the read
+    // immediately.
+    buffer->Append(Slice::FromCopiedBuffer(middle_->pending[peer_index()]));
+    middle_->pending[peer_index()].clear();
+    return true;
+  }
+}
+
+std::queue<size_t> FuzzingEventEngine::WriteSizesForConnection() {
+  if (write_sizes_for_future_connections_.empty()) return std::queue<size_t>();
+  auto ret = std::move(write_sizes_for_future_connections_.front());
+  write_sizes_for_future_connections_.pop();
+  return ret;
+}
+
+FuzzingEventEngine::EndpointMiddle::EndpointMiddle(int listener_port,
+                                                   int client_port)
+    : addrs{PortToAddress(listener_port), PortToAddress(client_port)},
+      write_sizes{g_fuzzing_event_engine->WriteSizesForConnection(),
+                  g_fuzzing_event_engine->WriteSizesForConnection()} {}
+
 EventEngine::ConnectionHandle FuzzingEventEngine::Connect(
-    OnConnectCallback, const ResolvedAddress&, const EndpointConfig&,
-    MemoryAllocator, Duration) {
-  abort();
+    OnConnectCallback on_connect, const ResolvedAddress& addr,
+    const EndpointConfig&, MemoryAllocator, Duration) {
+  // TODO(ctiller): do something with the timeout
+  // Schedule a timer to run (with some fuzzer selected delay) the on_connect
+  // callback.
+  auto task_handle = RunAfter(
+      Duration(0), [this, addr, on_connect = std::move(on_connect)]() mutable {
+        // Check for a legal address and extract the target port number.
+        auto port = ResolvedAddressGetPort(addr);
+        grpc_core::MutexLock lock(&*mu_);
+        // Find the listener that is listening on the target port.
+        for (auto it = listeners_.begin(); it != listeners_.end(); ++it) {
+          const auto& listener = *it;
+          // Listener must be started.
+          if (!listener->started) continue;
+          for (int listener_port : listener->ports) {
+            if (port == listener_port) {
+              // Port matches on a started listener: create an endpoint, call
+              // on_accept for the listener and on_connect for the client.
+              auto middle = std::make_shared<EndpointMiddle>(
+                  listener_port, g_fuzzing_event_engine->AllocatePort());
+              auto ep1 = std::make_unique<FuzzingEndpoint>(middle, 0);
+              auto ep2 = std::make_unique<FuzzingEndpoint>(middle, 1);
+              RunLocked([listener, ep1 = std::move(ep1)]() mutable {
+                listener->on_accept(
+                    std::move(ep1),
+                    listener->memory_allocator_factory->CreateMemoryAllocator(
+                        "fuzzing"));
+              });
+              RunLocked([on_connect = std::move(on_connect),
+                         ep2 = std::move(ep2)]() mutable {
+                on_connect(std::move(ep2));
+              });
+              return;
+            }
+          }
+        }
+        // Fail: no such listener.
+        RunLocked([on_connect = std::move(on_connect)]() mutable {
+          on_connect(absl::InvalidArgumentError("No listener found"));
+        });
+      });
+  return ConnectionHandle{{task_handle.keys[0], task_handle.keys[1]}};
 }
 
-bool FuzzingEventEngine::CancelConnect(ConnectionHandle) { abort(); }
+bool FuzzingEventEngine::CancelConnect(ConnectionHandle connection_handle) {
+  return Cancel(
+      TaskHandle{{connection_handle.keys[0], connection_handle.keys[1]}});
+}
 
 bool FuzzingEventEngine::IsWorkerThread() { abort(); }
 
@@ -167,7 +463,12 @@ EventEngine::TaskHandle FuzzingEventEngine::RunAfter(Duration when,
 
 EventEngine::TaskHandle FuzzingEventEngine::RunAfter(
     Duration when, absl::AnyInvocable<void()> closure) {
-  grpc_core::MutexLock lock(&mu_);
+  grpc_core::MutexLock lock(&*mu_);
+  return RunAfterLocked(when, std::move(closure));
+}
+
+EventEngine::TaskHandle FuzzingEventEngine::RunAfterLocked(
+    Duration when, absl::AnyInvocable<void()> closure) {
   const intptr_t id = next_task_id_;
   ++next_task_id_;
   const auto delay_it = task_delays_.find(id);
@@ -183,7 +484,7 @@ EventEngine::TaskHandle FuzzingEventEngine::RunAfter(
 }
 
 bool FuzzingEventEngine::Cancel(TaskHandle handle) {
-  grpc_core::MutexLock lock(&mu_);
+  grpc_core::MutexLock lock(&*mu_);
   GPR_ASSERT(handle.keys[1] == kTaskHandleSalt);
   const intptr_t id = handle.keys[0];
   auto it = tasks_by_id_.find(id);
@@ -202,22 +503,25 @@ gpr_timespec FuzzingEventEngine::GlobalNowImpl(gpr_clock_type clock_type) {
     return gpr_inf_future(clock_type);
   }
   GPR_ASSERT(g_fuzzing_event_engine != nullptr);
-  grpc_core::MutexLock lock(&g_fuzzing_event_engine->mu_);
+  grpc_core::MutexLock lock(&*mu_);
   return g_fuzzing_event_engine->NowAsTimespec(clock_type);
 }
 
-void FuzzingEventEngine::SetGlobalNowImplEngine(FuzzingEventEngine* engine) {
-  GPR_ASSERT(g_fuzzing_event_engine == nullptr);
-  g_fuzzing_event_engine = engine;
-  g_orig_gpr_now_impl = gpr_now_impl;
-  gpr_now_impl = GlobalNowImpl;
-}
-
-void FuzzingEventEngine::UnsetGlobalNowImplEngine(FuzzingEventEngine* engine) {
-  GPR_ASSERT(g_fuzzing_event_engine == engine);
+void FuzzingEventEngine::UnsetGlobalHooks() {
+  if (g_fuzzing_event_engine != this) return;
   g_fuzzing_event_engine = nullptr;
   gpr_now_impl = g_orig_gpr_now_impl;
   g_orig_gpr_now_impl = nullptr;
+  grpc_set_pick_port_functions(previous_pick_port_functions_);
+}
+
+FuzzingEventEngine::ListenerInfo::~ListenerInfo() {
+  GPR_ASSERT(g_fuzzing_event_engine != nullptr);
+  g_fuzzing_event_engine->Run(
+      [on_shutdown = std::move(on_shutdown),
+       shutdown_status = std::move(shutdown_status)]() mutable {
+        on_shutdown(std::move(shutdown_status));
+      });
 }
 
 }  // namespace experimental

test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h

@@ -15,30 +15,40 @@
 #ifndef GRPC_TEST_CORE_EVENT_ENGINE_FUZZING_EVENT_ENGINE_FUZZING_EVENT_ENGINE_H
 #define GRPC_TEST_CORE_EVENT_ENGINE_FUZZING_EVENT_ENGINE_FUZZING_EVENT_ENGINE_H
 
+#include <stddef.h>
+
 #include <chrono>
 #include <cstdint>
 #include <map>
 #include <memory>
+#include <queue>
 #include <ratio>
+#include <set>
 #include <utility>
+#include <vector>
 
 #include "absl/base/thread_annotations.h"
 #include "absl/functional/any_invocable.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
+#include "absl/types/optional.h"
 
 #include <grpc/event_engine/endpoint_config.h>
 #include <grpc/event_engine/event_engine.h>
 #include <grpc/event_engine/memory_allocator.h>
+#include <grpc/event_engine/slice_buffer.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/gprpp/no_destruct.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h"
+#include "test/core/util/port.h"
 
 namespace grpc_event_engine {
 namespace experimental {
 
 // EventEngine implementation to be used by fuzzers.
+// It's only allowed to have one FuzzingEventEngine instantiated at a time.
 class FuzzingEventEngine : public EventEngine {
  public:
   struct Options {
@@ -48,48 +58,54 @@ class FuzzingEventEngine : public EventEngine {
   };
   explicit FuzzingEventEngine(Options options,
                               const fuzzing_event_engine::Actions& actions);
-  ~FuzzingEventEngine() override = default;
+  ~FuzzingEventEngine() override { UnsetGlobalHooks(); }
 
-  void FuzzingDone();
-  void Tick();
+  // Once the fuzzing work is completed, this method should be called to speed
+  // quiescence.
+  void FuzzingDone() ABSL_LOCKS_EXCLUDED(mu_);
+  // Increment time once and perform any scheduled work.
+  void Tick() ABSL_LOCKS_EXCLUDED(mu_);
 
   absl::StatusOr<std::unique_ptr<Listener>> CreateListener(
       Listener::AcceptCallback on_accept,
       absl::AnyInvocable<void(absl::Status)> on_shutdown,
       const EndpointConfig& config,
       std::unique_ptr<MemoryAllocatorFactory> memory_allocator_factory)
-      override;
+      ABSL_LOCKS_EXCLUDED(mu_) override;
 
   ConnectionHandle Connect(OnConnectCallback on_connect,
                            const ResolvedAddress& addr,
                            const EndpointConfig& args,
-                           MemoryAllocator memory_allocator,
-                           Duration timeout) override;
+                           MemoryAllocator memory_allocator, Duration timeout)
+      ABSL_LOCKS_EXCLUDED(mu_) override;
 
-  bool CancelConnect(ConnectionHandle handle) override;
+  bool CancelConnect(ConnectionHandle handle) ABSL_LOCKS_EXCLUDED(mu_) override;
 
   bool IsWorkerThread() override;
 
   std::unique_ptr<DNSResolver> GetDNSResolver(
       const DNSResolver::ResolverOptions& options) override;
 
-  void Run(Closure* closure) override;
-  void Run(absl::AnyInvocable<void()> closure) override;
-  TaskHandle RunAfter(Duration when, Closure* closure) override;
-  TaskHandle RunAfter(Duration when,
-                      absl::AnyInvocable<void()> closure) override;
-  bool Cancel(TaskHandle handle) override;
+  void Run(Closure* closure) ABSL_LOCKS_EXCLUDED(mu_) override;
+  void Run(absl::AnyInvocable<void()> closure)
+      ABSL_LOCKS_EXCLUDED(mu_) override;
+  TaskHandle RunAfter(Duration when, Closure* closure)
+      ABSL_LOCKS_EXCLUDED(mu_) override;
+  TaskHandle RunAfter(Duration when, absl::AnyInvocable<void()> closure)
+      ABSL_LOCKS_EXCLUDED(mu_) override;
+  bool Cancel(TaskHandle handle) ABSL_LOCKS_EXCLUDED(mu_) override;
 
   using Time = std::chrono::time_point<FuzzingEventEngine, Duration>;
 
   Time Now() ABSL_LOCKS_EXCLUDED(mu_);
 
-  static void SetGlobalNowImplEngine(FuzzingEventEngine* engine)
-      ABSL_LOCKS_EXCLUDED(mu_);
-  static void UnsetGlobalNowImplEngine(FuzzingEventEngine* engine)
-      ABSL_LOCKS_EXCLUDED(mu_);
+  // Clear any global hooks installed by this event engine. Call prior to
+  // destruction to ensure no overlap between tests if constructing/destructing
+  // each test.
+  void UnsetGlobalHooks() ABSL_LOCKS_EXCLUDED(mu_);
 
  private:
+  // One pending task to be run.
   struct Task {
     Task(intptr_t id, absl::AnyInvocable<void()> closure)
         : id(id), closure(std::move(closure)) {}
@@ -97,13 +113,140 @@ class FuzzingEventEngine : public EventEngine {
     absl::AnyInvocable<void()> closure;
   };
 
+  // Per listener information.
+  // We keep a shared_ptr to this, one reference held by the FuzzingListener
+  // Listener implementation, and one reference in the event engine state, so it
+  // may be iterated through and inspected - principally to discover the ports
+  // on which this listener is listening.
+  struct ListenerInfo {
+    ListenerInfo(
+        Listener::AcceptCallback on_accept,
+        absl::AnyInvocable<void(absl::Status)> on_shutdown,
+        std::unique_ptr<MemoryAllocatorFactory> memory_allocator_factory)
+        : on_accept(std::move(on_accept)),
+          on_shutdown(std::move(on_shutdown)),
+          memory_allocator_factory(std::move(memory_allocator_factory)),
+          started(false) {}
+    ~ListenerInfo() ABSL_LOCKS_EXCLUDED(mu_);
+    // The callback to invoke when a new connection is accepted.
+    Listener::AcceptCallback on_accept;
+    // The callback to invoke when the listener is shut down.
+    absl::AnyInvocable<void(absl::Status)> on_shutdown;
+    // The memory allocator factory to use for this listener.
+    const std::unique_ptr<MemoryAllocatorFactory> memory_allocator_factory;
+    // The ports on which this listener is listening.
+    std::vector<int> ports ABSL_GUARDED_BY(mu_);
+    // Has start been called on the listener?
+    // Used to emulate the Bind/Start semantics demanded by the API.
+    bool started ABSL_GUARDED_BY(mu_);
+    // The status to return via on_shutdown.
+    absl::Status shutdown_status ABSL_GUARDED_BY(mu_);
+  };
+
+  // Implementation of Listener.
+  class FuzzingListener final : public Listener {
+   public:
+    explicit FuzzingListener(std::shared_ptr<ListenerInfo> info)
+        : info_(std::move(info)) {}
+    ~FuzzingListener() override;
+    absl::StatusOr<int> Bind(const ResolvedAddress& addr) override;
+    absl::Status Start() override;
+
+   private:
+    std::shared_ptr<ListenerInfo> info_;
+  };
+
+  // One read that's outstanding.
+  struct PendingRead {
+    // Callback to invoke when the read completes.
+    absl::AnyInvocable<void(absl::Status)> on_read;
+    // The buffer to read into.
+    SliceBuffer* buffer;
+  };
+
+  // The join between two Endpoint instances.
+  struct EndpointMiddle {
+    EndpointMiddle(int listener_port, int client_port)
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+    // Address of each side of the endpoint.
+    const ResolvedAddress addrs[2];
+    // Is the endpoint closed?
+    bool closed ABSL_GUARDED_BY(mu_) = false;
+    // Bytes written into each endpoint and awaiting a read.
+    std::vector<uint8_t> pending[2] ABSL_GUARDED_BY(mu_);
+    // The sizes of each accepted write, as determined by the fuzzer actions.
+    std::queue<size_t> write_sizes[2] ABSL_GUARDED_BY(mu_);
+    // The next read that's pending (or nullopt).
+    absl::optional<PendingRead> pending_read[2] ABSL_GUARDED_BY(mu_);
+
+    // Helper to take some bytes from data and queue them into pending[index].
+    // Returns true if all bytes were consumed, false if more writes are needed.
+    bool Write(SliceBuffer* data, int index) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+  };
+
+  // Implementation of Endpoint.
+  // When a connection is formed, we create two of these - one with index 0, the
+  // other index 1, both pointing to the same EndpointMiddle.
+  class FuzzingEndpoint final : public Endpoint {
+   public:
+    FuzzingEndpoint(std::shared_ptr<EndpointMiddle> middle, int index)
+        : middle_(std::move(middle)), index_(index) {}
+    ~FuzzingEndpoint() override;
+
+    bool Read(absl::AnyInvocable<void(absl::Status)> on_read,
+              SliceBuffer* buffer, const ReadArgs* args) override;
+    bool Write(absl::AnyInvocable<void(absl::Status)> on_writable,
+               SliceBuffer* data, const WriteArgs* args) override;
+    const ResolvedAddress& GetPeerAddress() const override {
+      return middle_->addrs[peer_index()];
+    }
+    const ResolvedAddress& GetLocalAddress() const override {
+      return middle_->addrs[my_index()];
+    }
+
+   private:
+    int my_index() const { return index_; }
+    int peer_index() const { return 1 - index_; }
+    // Schedule additional writes to be performed later.
+    // Takes a ref to middle instead of holding this, so that should the
+    // endpoint be destroyed we don't have to worry about use-after-free.
+    // Instead that scheduled callback will see the middle is closed and finally
+    // report completion to the caller.
+    // Since there is no timeliness contract for the completion of writes after
+    // endpoint shutdown, it's believed this is a legal implementation.
+    static void ScheduleDelayedWrite(
+        std::shared_ptr<EndpointMiddle> middle, int index,
+        absl::AnyInvocable<void(absl::Status)> on_writable, SliceBuffer* data)
+        ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+    const std::shared_ptr<EndpointMiddle> middle_;
+    const int index_;
+  };
+
+  void RunLocked(absl::AnyInvocable<void()> closure)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+    RunAfterLocked(Duration::zero(), std::move(closure));
+  }
+
+  TaskHandle RunAfterLocked(Duration when, absl::AnyInvocable<void()> closure)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+
+  // Allocate a port. Considered fuzzer selected port orderings first, and then
+  // falls back to an exhaustive incremental search from port #1.
+  int AllocatePort() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+  // Is the given port in use by any listener?
+  bool IsPortUsed(int port) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+  // For the next connection being built, query the list of fuzzer selected
+  // write size limits.
+  std::queue<size_t> WriteSizesForConnection()
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
+
   gpr_timespec NowAsTimespec(gpr_clock_type clock_type)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_);
   static gpr_timespec GlobalNowImpl(gpr_clock_type clock_type)
       ABSL_LOCKS_EXCLUDED(mu_);
   const Duration final_tick_length_;
 
-  grpc_core::Mutex mu_;
+  static grpc_core::NoDestruct<grpc_core::Mutex> mu_;
 
   intptr_t next_task_id_ ABSL_GUARDED_BY(mu_);
   intptr_t current_tick_ ABSL_GUARDED_BY(mu_);
@@ -113,6 +256,18 @@ class FuzzingEventEngine : public EventEngine {
   std::map<intptr_t, std::shared_ptr<Task>> tasks_by_id_ ABSL_GUARDED_BY(mu_);
   std::multimap<Time, std::shared_ptr<Task>> tasks_by_time_
       ABSL_GUARDED_BY(mu_);
+  std::set<std::shared_ptr<ListenerInfo>> listeners_ ABSL_GUARDED_BY(mu_);
+  // Fuzzer selected port allocations.
+  std::queue<int> free_ports_ ABSL_GUARDED_BY(mu_);
+  // Next free port to allocate once fuzzer selections are exhausted.
+  int next_free_port_ ABSL_GUARDED_BY(mu_) = 1;
+  // Ports that were included in the fuzzer selected port orderings.
+  std::set<int> fuzzer_mentioned_ports_ ABSL_GUARDED_BY(mu_);
+  // Fuzzer selected write sizes for future connections - one picked off per
+  // WriteSizesForConnection() call.
+  std::queue<std::queue<size_t>> write_sizes_for_future_connections_
+      ABSL_GUARDED_BY(mu_);
+  grpc_pick_port_functions previous_pick_port_functions_;
 };
 
 }  // namespace experimental

test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.proto

@@ -21,9 +21,18 @@ message Actions {
     repeated Delay tick_lengths = 1;
     // Map of task id to the amount to delay execution of the task by.
     repeated Delay run_delay = 2;
+    // Order in which to bind port numbers.
+    // After this ports are assigned in order, from 1 to 65535.
+    repeated uint32 assign_ports = 3;
+    // Write size constraints for each connection, in order of creation.
+    repeated Connection connections = 4;
 };
 
 message Delay {
     uint32 id = 1;
     uint64 delay_us = 2;
 };
+
+message Connection {
+    repeated uint32 write_size = 1;
+};

test/core/event_engine/test_suite/BUILD

@@ -115,6 +115,8 @@ grpc_cc_test(
     uses_polling = False,
     deps = [
         "//test/core/event_engine/fuzzing_event_engine",
+        "//test/core/event_engine/test_suite/tests:client",
+        "//test/core/event_engine/test_suite/tests:server",
         "//test/core/event_engine/test_suite/tests:timer",
     ],
 )

test/core/event_engine/test_suite/event_engine_test_framework.cc

@@ -18,19 +18,19 @@
 #include <grpc/event_engine/event_engine.h>
 
 absl::AnyInvocable<
-    std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>*
+    std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>*
     g_ee_factory = nullptr;
 
 absl::AnyInvocable<
-    std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>*
+    std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>*
     g_oracle_ee_factory = nullptr;
 
 void SetEventEngineFactories(
     absl::AnyInvocable<
-        std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>
+        std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>
         factory,
     absl::AnyInvocable<
-        std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>
+        std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>
         oracle_ee_factory) {
   testing::AddGlobalTestEnvironment(new EventEngineTestEnvironment(
       std::move(factory), std::move(oracle_ee_factory)));

test/core/event_engine/test_suite/event_engine_test_framework.h

@@ -25,11 +25,11 @@
 #include <grpc/support/log.h>
 
 extern absl::AnyInvocable<
-    std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>*
+    std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>*
     g_ee_factory;
 
 extern absl::AnyInvocable<
-    std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>*
+    std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>*
     g_oracle_ee_factory;
 
 // Manages the lifetime of the global EventEngine factory.
@@ -37,10 +37,10 @@ class EventEngineTestEnvironment : public testing::Environment {
  public:
   EventEngineTestEnvironment(
       absl::AnyInvocable<
-          std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>
+          std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>
           factory,
       absl::AnyInvocable<
-          std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>
+          std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>
           oracle_factory)
       : factory_(std::move(factory)),
         oracle_factory_(std::move(oracle_factory)) {}
@@ -57,22 +57,22 @@ class EventEngineTestEnvironment : public testing::Environment {
 
  private:
   absl::AnyInvocable<
-      std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>
+      std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>
       factory_;
   absl::AnyInvocable<
-      std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>
+      std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>
       oracle_factory_;
 };
 
 class EventEngineTest : public testing::Test {
  protected:
-  std::unique_ptr<grpc_event_engine::experimental::EventEngine>
+  std::shared_ptr<grpc_event_engine::experimental::EventEngine>
   NewEventEngine() {
     GPR_ASSERT(g_ee_factory != nullptr);
     return (*g_ee_factory)();
   }
 
-  std::unique_ptr<grpc_event_engine::experimental::EventEngine>
+  std::shared_ptr<grpc_event_engine::experimental::EventEngine>
   NewOracleEventEngine() {
     GPR_ASSERT(g_oracle_ee_factory != nullptr);
     return (*g_oracle_ee_factory)();
@@ -83,10 +83,10 @@ class EventEngineTest : public testing::Test {
 // EventEngine can additionally be specified here.
 void SetEventEngineFactories(
     absl::AnyInvocable<
-        std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>
+        std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>
         ee_factory,
     absl::AnyInvocable<
-        std::unique_ptr<grpc_event_engine::experimental::EventEngine>()>
+        std::shared_ptr<grpc_event_engine::experimental::EventEngine>()>
         oracle_ee_factory);
 
 #endif  // GRPC_TEST_CORE_EVENT_ENGINE_TEST_SUITE_EVENT_ENGINE_TEST_FRAMEWORK_H

test/core/event_engine/test_suite/fuzzing_event_engine_test.cc

@@ -67,12 +67,11 @@ class ThreadedFuzzingEventEngine : public FuzzingEventEngine {
 
 int main(int argc, char** argv) {
   testing::InitGoogleTest(&argc, argv);
-  SetEventEngineFactories(
-      []() {
-        return std::make_unique<
-            grpc_event_engine::experimental::ThreadedFuzzingEventEngine>();
-      },
-      nullptr);
+  std::shared_ptr<grpc_event_engine::experimental::FuzzingEventEngine> engine =
+      std::make_shared<
+          grpc_event_engine::experimental::ThreadedFuzzingEventEngine>();
+  SetEventEngineFactories([engine]() { return engine; },
+                          [engine]() { return engine; });
   grpc_event_engine::experimental::InitTimerTests();
   return RUN_ALL_TESTS();
 }

test/core/event_engine/test_suite/posix_event_engine_test.cc

@@ -15,7 +15,6 @@
 
 #include <gtest/gtest.h>
 
-#include <grpc/event_engine/event_engine.h>
 #include <grpc/grpc.h>
 
 #include "src/core/lib/event_engine/posix_engine/posix_engine.h"

test/core/event_engine/test_suite/tests/client_test.cc

@@ -69,7 +69,6 @@ using Endpoint = ::grpc_event_engine::experimental::EventEngine::Endpoint;
 using Listener = ::grpc_event_engine::experimental::EventEngine::Listener;
 using ::grpc_event_engine::experimental::GetNextSendMessage;
 using ::grpc_event_engine::experimental::NotifyOnDelete;
-using ::grpc_event_engine::experimental::WaitForSingleOwner;
 
 constexpr int kNumExchangedMessages = 100;
 
@@ -96,7 +95,6 @@ TEST_F(EventEngineClientTest, ConnectToNonExistentListenerTest) {
       *URIToResolvedAddress(target_addr), config,
       memory_quota->CreateMemoryAllocator("conn-1"), 24h);
   signal.WaitForNotification();
-  WaitForSingleOwner(std::move(test_ee));
 }
 
 // Create a connection using the test EventEngine to a listener created
@@ -171,7 +169,6 @@ TEST_F(EventEngineClientTest, ConnectExchangeBidiDataTransferTest) {
   client_endpoint.reset();
   server_endpoint.reset();
   listener.reset();
-  WaitForSingleOwner(std::move(test_ee));
 }
 
 // Create 1 listener bound to N IPv6 addresses and M connections where M > N and
@@ -298,7 +295,6 @@ TEST_F(EventEngineClientTest, MultipleIPv6ConnectionsToOneOracleListenerTest) {
     t.join();
   }
   server_endpoint.reset();
-  WaitForSingleOwner(std::move(test_ee));
 }
 
 // TODO(vigneshbabu): Add more tests which create listeners bound to a mix

test/core/event_engine/test_suite/tests/server_test.cc

@@ -68,7 +68,6 @@ using ::grpc_event_engine::experimental::URIToResolvedAddress;
 using Endpoint = ::grpc_event_engine::experimental::EventEngine::Endpoint;
 using Listener = ::grpc_event_engine::experimental::EventEngine::Listener;
 using ::grpc_event_engine::experimental::GetNextSendMessage;
-using ::grpc_event_engine::experimental::WaitForSingleOwner;
 
 constexpr int kNumExchangedMessages = 100;
 
@@ -84,9 +83,11 @@ TEST_F(EventEngineServerTest, CannotBindAfterStarted) {
   // Bind an initial port to ensure normal listener startup
   auto resolved_addr = URIToResolvedAddress(absl::StrCat(
       "ipv6:[::1]:", std::to_string(grpc_pick_unused_port_or_die())));
-  ASSERT_TRUE(resolved_addr.ok());
-  ASSERT_TRUE((*listener)->Bind(*resolved_addr).ok());
-  ASSERT_TRUE((*listener)->Start().ok());
+  ASSERT_TRUE(resolved_addr.ok()) << resolved_addr.status();
+  auto bind_result = (*listener)->Bind(*resolved_addr);
+  ASSERT_TRUE(bind_result.ok()) << bind_result.status();
+  auto listen_result = (*listener)->Start();
+  ASSERT_TRUE(listen_result.ok()) << listen_result;
   // A subsequent bind, which should fail
   auto resolved_addr2 = URIToResolvedAddress(absl::StrCat(
       "ipv6:[::1]:", std::to_string(grpc_pick_unused_port_or_die())));
@@ -137,7 +138,7 @@ TEST_F(EventEngineServerTest, ServerConnectExchangeBidiDataTransferTest) {
   oracle_ee->Connect(
       [&client_endpoint,
        &client_signal](absl::StatusOr<std::unique_ptr<Endpoint>> endpoint) {
-        ASSERT_TRUE(endpoint.ok());
+        ASSERT_TRUE(endpoint.ok()) << endpoint.status();
         client_endpoint = std::move(*endpoint);
         client_signal.Notify();
       },
@@ -165,7 +166,6 @@ TEST_F(EventEngineServerTest, ServerConnectExchangeBidiDataTransferTest) {
   client_endpoint.reset();
   server_endpoint.reset();
   listener.reset();
-  WaitForSingleOwner(std::move(test_ee));
 }
 
 // Create 1 listener bound to N IPv6 addresses and M connections where M > N and
@@ -294,7 +294,6 @@ TEST_F(EventEngineServerTest,
   }
   server_endpoint.reset();
   listener.reset();
-  WaitForSingleOwner(std::move(test_ee));
 }
 
 // TODO(vigneshbabu): Add more tests which create listeners bound to a mix

test/core/event_engine/test_suite/thready_posix_event_engine_test.cc

@@ -15,7 +15,6 @@
 
 #include <gtest/gtest.h>
 
-#include <grpc/event_engine/event_engine.h>
 #include <grpc/grpc.h>
 
 #include "src/core/lib/event_engine/posix_engine/posix_engine.h"

test/core/util/BUILD

@@ -151,6 +151,7 @@ grpc_cc_library(
         "//src/core:time",
         "//test/core/event_engine:test_init",
     ],
+    alwayslink = 1,
 )
 
 grpc_cc_library(
@@ -190,6 +191,7 @@ grpc_cc_library(
         "//src/core:time",
         "//test/core/event_engine:test_init",
     ],
+    alwayslink = 1,
 )
 
 grpc_cc_test(
@@ -342,6 +344,24 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "fake_udp_and_tcp_server_unsecure",
+    srcs = ["fake_udp_and_tcp_server.cc"],
+    hdrs = ["fake_udp_and_tcp_server.h"],
+    external_deps = [
+        "absl/status:statusor",
+        "absl/strings",
+    ],
+    language = "C++",
+    deps = [
+        "grpc_test_util_unsecure",
+        "//:gpr",
+        "//:grpc",
+        "//:sockaddr_utils",
+        "//src/core:resolved_address",
+    ],
+)
+
 grpc_cc_library(
     name = "build",
     srcs = ["build.cc"],

test/core/util/port.cc

@@ -18,20 +18,17 @@
 
 #include "src/core/lib/iomgr/port.h"
 
+#include <stdio.h>
 #include <stdlib.h>
 
-#include <grpc/support/sync.h>
-
-#include "src/core/lib/gprpp/sync.h"
-#include "test/core/util/test_config.h"
-#if defined(GRPC_TEST_PICK_PORT)
-
-#include <stdio.h>
+#include <utility>
 
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
+#include <grpc/support/sync.h>
 
+#include "src/core/lib/gprpp/sync.h"
 #include "test/core/util/port.h"
 #include "test/core/util/port_server_client.h"
 
@@ -98,7 +95,7 @@ static int grpc_pick_unused_port_impl(void) {
 }
 
 static int grpc_pick_unused_port_or_die_impl(void) {
-  int port = grpc_pick_unused_port();
+  int port = grpc_pick_unused_port_impl();
   if (port == 0) {
     fprintf(stderr,
             "gRPC tests require a helper port server to allocate ports used \n"
@@ -116,27 +113,25 @@ static void grpc_recycle_unused_port_impl(int port) {
   GPR_ASSERT(free_chosen_port_locked(port));
 }
 
-static grpc_pick_port_functions g_pick_port_functions = {
-    grpc_pick_unused_port_impl, grpc_pick_unused_port_or_die_impl,
-    grpc_recycle_unused_port_impl};
-
-int grpc_pick_unused_port(void) {
-  return g_pick_port_functions.pick_unused_port_fn();
+namespace {
+grpc_pick_port_functions& functions() {
+  static grpc_pick_port_functions* functions = new grpc_pick_port_functions{
+      grpc_pick_unused_port_or_die_impl, grpc_recycle_unused_port_impl};
+  return *functions;
 }
+}  // namespace
 
 int grpc_pick_unused_port_or_die(void) {
-  return g_pick_port_functions.pick_unused_port_or_die_fn();
+  return functions().pick_unused_port_or_die_fn();
 }
 
 void grpc_recycle_unused_port(int port) {
-  g_pick_port_functions.recycle_unused_port_fn(port);
+  functions().recycle_unused_port_fn(port);
 }
 
-void grpc_set_pick_port_functions(grpc_pick_port_functions functions) {
-  GPR_ASSERT(functions.pick_unused_port_fn != nullptr);
-  GPR_ASSERT(functions.pick_unused_port_or_die_fn != nullptr);
-  GPR_ASSERT(functions.recycle_unused_port_fn != nullptr);
-  g_pick_port_functions = functions;
+grpc_pick_port_functions grpc_set_pick_port_functions(
+    grpc_pick_port_functions new_functions) {
+  GPR_ASSERT(new_functions.pick_unused_port_or_die_fn != nullptr);
+  GPR_ASSERT(new_functions.recycle_unused_port_fn != nullptr);
+  return std::exchange(functions(), new_functions);
 }
-
-#endif  // GRPC_TEST_PICK_PORT

test/core/util/port.h

@@ -20,14 +20,10 @@
 #define GRPC_TEST_CORE_UTIL_PORT_H
 
 typedef struct grpc_pick_port_functions {
-  int (*pick_unused_port_fn)(void);
   int (*pick_unused_port_or_die_fn)(void);
   void (*recycle_unused_port_fn)(int port);
 } grpc_pick_port_functions;
 
-// pick a port number that is currently unused by either tcp or udp. return
-// 0 on failure.
-int grpc_pick_unused_port(void);
 // pick a port number that is currently unused by either tcp or udp. abort
 // on failure.
 int grpc_pick_unused_port_or_die(void);
@@ -39,6 +35,8 @@ int grpc_pick_unused_port_or_die(void);
 void grpc_recycle_unused_port(int port);
 
 /// Request the family of pick_port functions in \a functions be used.
-void grpc_set_pick_port_functions(grpc_pick_port_functions functions);
+/// Returns the current set so they can be restored later.
+grpc_pick_port_functions grpc_set_pick_port_functions(
+    grpc_pick_port_functions functions);
 
 #endif  // GRPC_TEST_CORE_UTIL_PORT_H

test/core/util/port_isolated_runtime_environment.cc

@@ -54,7 +54,7 @@ static int grpc_pick_unused_port_or_die_impl(void) {
          (s_initial_offset + orig_counter_val) % (MAX_PORT - MIN_PORT + 1);
 }
 
-int grpc_pick_unused_port_or_die(void) {
+static int isolated_pick_unused_port_or_die(void) {
   while (true) {
     int port = grpc_pick_unused_port_or_die_impl();
     // 5985 cannot be bound on Windows RBE and results in
@@ -67,6 +67,13 @@ int grpc_pick_unused_port_or_die(void) {
   }
 }
 
-void grpc_recycle_unused_port(int port) { (void)port; }
+static void isolated_recycle_unused_port(int port) { (void)port; }
+
+// We don't actually use prev_fns for anything, but need to save it in order to
+// be able to call grpc_set_pick_port_functions() to override defaults for this
+// environment.
+static const auto prev_fns =
+    grpc_set_pick_port_functions(grpc_pick_port_functions{
+        isolated_pick_unused_port_or_die, isolated_recycle_unused_port});
 
 #endif  // GRPC_PORT_ISOLATED_RUNTIME

test/core/util/port_server_client.cc

@@ -18,8 +18,11 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "test/core/util/port_server_client.h"
+
 #include <stdint.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include <cmath>
 #include <initializer_list>
@@ -28,11 +31,20 @@
 #include <utility>
 
 #include "absl/status/statusor.h"
+#include "absl/strings/str_format.h"
+
+#include <grpc/grpc.h>
+#include <grpc/grpc_security.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/time.h>
 
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/status_helper.h"
 #include "src/core/lib/gprpp/time.h"
+#include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/error.h"
@@ -40,24 +52,8 @@
 #include "src/core/lib/iomgr/iomgr_fwd.h"
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/pollset.h"
-#include "src/core/lib/uri/uri_parser.h"
-#include "test/core/util/test_config.h"
-
-#ifdef GRPC_TEST_PICK_PORT
-#include <string.h>
-
-#include "absl/strings/str_format.h"
-
-#include <grpc/grpc.h>
-#include <grpc/grpc_security.h>
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/sync.h>
-#include <grpc/support/time.h>
-
-#include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "test/core/util/port_server_client.h"
+#include "src/core/lib/uri/uri_parser.h"
 
 typedef struct freereq {
   gpr_mu* mu = nullptr;
@@ -271,5 +267,3 @@ int grpc_pick_port_using_server(void) {
 
   return pr.port;
 }
-
-#endif  // GRPC_TEST_PICK_PORT

test/cpp/naming/generate_resolver_component_tests.bzl

@@ -54,7 +54,7 @@ def generate_resolver_component_tests():
             deps = [
                 "//test/cpp/util:test_util%s" % unsecure_build_config_suffix,
                 "//test/core/util:grpc_test_util%s" % unsecure_build_config_suffix,
-                "//test/core/util:fake_udp_and_tcp_server",
+                "//test/core/util:fake_udp_and_tcp_server%s" % unsecure_build_config_suffix,
                 "//:grpc++%s" % unsecure_build_config_suffix,
                 "//:grpc%s" % unsecure_build_config_suffix,
                 "//:gpr",
@@ -71,10 +71,10 @@ def generate_resolver_component_tests():
                 "absl/flags:flag",
             ],
             deps = [
-                "//test/cpp/util:test_util",
-                "//test/core/util:grpc_test_util",
-                "//:grpc++",
-                "//:grpc",
+                "//test/cpp/util:test_util%s" % unsecure_build_config_suffix,
+                "//test/core/util:grpc_test_util%s" % unsecure_build_config_suffix,
+                "//:grpc++%s" % unsecure_build_config_suffix,
+                "//:grpc%s" % unsecure_build_config_suffix,
                 "//:gpr",
                 "//test/cpp/util:test_config",
             ],