[test] Re-land: Enable EventEngine experiments for Posix end2end tests (#32948)

Relands #32844.

End2end tests will now wait for the default EventEngine to shut down
between tests. This should avoid some use-after-frees and leaks.

CMakeLists.txt

@@ -9320,6 +9320,7 @@ add_executable(core_end2end_tests
   test/core/end2end/tests/trailing_metadata.cc
   test/core/end2end/tests/write_buffering.cc
   test/core/end2end/tests/write_buffering_at_end.cc
+  test/core/event_engine/event_engine_test_utils.cc
   test/core/util/test_lb_policies.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
@@ -13038,6 +13039,7 @@ add_executable(h2_ssl_cert_test
   test/core/end2end/end2end_tests.cc
   test/core/end2end/fixtures/local_util.cc
   test/core/end2end/h2_ssl_cert_test.cc
+  test/core/event_engine/event_engine_test_utils.cc
   third_party/googletest/googletest/src/gtest-all.cc
   third_party/googletest/googlemock/src/gmock-all.cc
 )
@@ -13066,6 +13068,7 @@ target_link_libraries(h2_ssl_cert_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ZLIB_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc_unsecure
   grpc_test_util
 )
 

bazel/experiments.bzl

@@ -24,6 +24,8 @@ EXPERIMENTS = {
             "transport_supplies_client_latency",
         ],
         "core_end2end_test": [
+            "event_engine_client",
+            "event_engine_listener",
             "promise_based_client_call",
             "promise_based_server_call",
         ],

build_autogenerated.yaml

@@ -6166,6 +6166,7 @@ targets:
   - test/core/end2end/fixtures/secure_fixture.h
   - test/core/end2end/fixtures/sockpair_fixture.h
   - test/core/end2end/tests/cancel_test_helpers.h
+  - test/core/event_engine/event_engine_test_utils.h
   - test/core/util/test_lb_policies.h
   src:
   - test/core/end2end/cq_verifier.cc
@@ -6263,6 +6264,7 @@ targets:
   - test/core/end2end/tests/trailing_metadata.cc
   - test/core/end2end/tests/write_buffering.cc
   - test/core/end2end/tests/write_buffering_at_end.cc
+  - test/core/event_engine/event_engine_test_utils.cc
   - test/core/util/test_lb_policies.cc
   deps:
   - grpc_authorization_provider
@@ -8486,6 +8488,7 @@ targets:
   - test/core/end2end/fixtures/secure_fixture.h
   - test/core/end2end/fixtures/sockpair_fixture.h
   - test/core/end2end/tests/cancel_test_helpers.h
+  - test/core/event_engine/event_engine_test_utils.h
   src:
   - test/core/end2end/cq_verifier.cc
   - test/core/end2end/data/client_certs.cc
@@ -8495,7 +8498,9 @@ targets:
   - test/core/end2end/end2end_tests.cc
   - test/core/end2end/fixtures/local_util.cc
   - test/core/end2end/h2_ssl_cert_test.cc
+  - test/core/event_engine/event_engine_test_utils.cc
   deps:
+  - grpc_unsecure
   - grpc_test_util
 - name: h2_ssl_session_reuse_test
   gtest: true

src/core/lib/experiments/experiments.yaml

@@ -29,7 +29,7 @@
 #              that that test should be run with this experiment enabled in CI
 #
 # Well known test tags:
-#   core_end2end_tests:  all tests, fixtures in the core end2end suite
+#   core_end2end_test:   all tests, fixtures in the core end2end suite
 #   endpoint_test:       endpoint related iomgr tests
 #   flow_control_test:   tests pertaining explicitly to flow control
 #   hpack_test:          hpack encode/decode tests
@@ -47,8 +47,7 @@
   owner: vigneshbabu@google.com
   test_tags: ["endpoint_test", "flow_control_test"]
 - name: tcp_rcv_lowat
-  description:
+  description: Use SO_RCVLOWAT to avoid wakeups on the read path.
-    Use SO_RCVLOWAT to avoid wakeups on the read path.
   default: false
   expiry: 2023/03/01
   owner: vigneshbabu@google.com
@@ -63,36 +62,31 @@
   owner: vigneshbabu@google.com
   test_tags: ["flow_control_test"]
 - name: flow_control_fixes
-  description:
+  description: Various fixes for flow control, max frame size setting.
-    Various fixes for flow control, max frame size setting.
   default: true
   expiry: 2023/04/01
   owner: ctiller@google.com
   test_tags: ["flow_control_test"]
 - name: memory_pressure_controller
-  description:
+  description: New memory pressure controller
-    New memory pressure controller
   default: false
   expiry: 2023/03/01
   owner: ctiller@google.com
   test_tags: [resource_quota_test]
 - name: unconstrained_max_quota_buffer_size
-  description:
+  description: Discard the cap on the max free pool size for one memory allocator
-    Discard the cap on the max free pool size for one memory allocator
   default: false
   expiry: 2023/03/01
   owner: ctiller@google.com
   test_tags: [resource_quota_test]
 - name: event_engine_client
-  description:
+  description: Use EventEngine clients instead of iomgr's grpc_tcp_client
-    Use EventEngine clients instead of iomgr's grpc_tcp_client
   default: false
   expiry: 2023/04/13
   owner: hork@google.com
-  test_tags: ["event_engine_client_test"]
+  test_tags: ["core_end2end_test", "event_engine_client_test"]
 - name: monitoring_experiment
-  description:
+  description: Placeholder experiment to prove/disprove our monitoring is working
-    Placeholder experiment to prove/disprove our monitoring is working
   default: true
   expiry: 2023/06/01
   owner: ctiller@google.com
@@ -106,8 +100,7 @@
   owner: ctiller@google.com
   test_tags: ["core_end2end_test", "lame_client_test"]
 - name: free_large_allocator
-  description:
+  description: If set, return all free bytes from a "big" allocator
-    If set, return all free bytes from a "big" allocator
   default: false
   expiry: 2023/04/01
   owner: alishananda@google.com
@@ -123,29 +116,25 @@
   # removing these until I can stabilize tests a little further
   disabled_test_tags: ["cpp_end2end_test", "xds_end2end_test"]
 - name: transport_supplies_client_latency
-  description:
+  description: If set, use the transport represented value for client latency in opencensus
-    If set, use the transport represented value for client latency in opencensus
   default: false
   expiry: 2023/06/01
   owner: ctiller@google.com
   test_tags: [census_test]
 - name: event_engine_listener
-  description:
+  description: Use EventEngine listeners instead of iomgr's grpc_tcp_server
-    Use EventEngine listeners instead of iomgr's grpc_tcp_server
   default: false
   expiry: 2023/05/13
   owner: vigneshbabu@google.com
-  test_tags: ["event_engine_listener_test"]
+  test_tags: ["core_end2end_test", "event_engine_listener_test"]
 - name: schedule_cancellation_over_write
-  description:
+  description: Allow cancellation op to be scheduled over a write
-    Allow cancellation op to be scheduled over a write
   default: false
   expiry: 2023/07/01
   owner: vigneshbabu@google.com
   test_tags: []
 - name: trace_record_callops
-  description:
+  description: Enables tracing of call batch initiation and completion.
-    Enables tracing of call batch initiation and completion.
   default: false
   expiry: 2023/07/01
   owner: vigneshbabu@google.com

test/core/end2end/BUILD

@@ -127,9 +127,11 @@ grpc_cc_library(
         "//:grpc_public_hdrs",
         "//src/core:bitset",
         "//src/core:channel_args",
+        "//src/core:default_event_engine",
         "//src/core:no_destruct",
         "//src/core:slice",
         "//src/core:time",
+        "//test/core/event_engine:event_engine_test_utils",
         "//test/core/util:grpc_test_util",
     ],
 )
@@ -318,6 +320,7 @@ grpc_cc_test(
         "//src/core:channel_stack_type",
         "//src/core:closure",
         "//src/core:error",
+        "//src/core:experiments",
         "//src/core:grpc_authorization_base",
         "//src/core:grpc_fake_credentials",
         "//src/core:iomgr_port",

test/core/end2end/end2end_test_main.cc

@@ -44,6 +44,7 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/config/config_vars.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/gprpp/host_port.h"
 #include "src/core/lib/gprpp/no_destruct.h"
 #include "src/core/lib/iomgr/error.h"
@@ -924,6 +925,24 @@ class ConfigQuery {
         AllConfigs());
     std::vector<const CoreTestConfiguration*> out;
     for (const CoreTestConfiguration& config : *kConfigs) {
+      if (IsEventEngineClientEnabled() &&
+          // Ignore disabled test suites
+          (((exclude_features_ &
+             FEATURE_MASK_DISABLE_EVENT_ENGINE_CLIENT_EXPERIMENT) != 0) ||
+           // Ignore disabled tests
+           ((config.feature_mask &
+             FEATURE_MASK_DISABLE_EVENT_ENGINE_CLIENT_EXPERIMENT) != 0))) {
+        continue;
+      }
+      if (IsEventEngineListenerEnabled() &&
+          // Ignore disabled test suites
+          (((exclude_features_ &
+             FEATURE_MASK_DISABLE_EVENT_ENGINE_LISTENER_EXPERIMENT) != 0) ||
+           // Ignore disabled tests
+           ((config.feature_mask &
+             FEATURE_MASK_DISABLE_EVENT_ENGINE_LISTENER_EXPERIMENT) != 0))) {
+        continue;
+      }
       if ((config.feature_mask & enforce_features_) == enforce_features_ &&
           (config.feature_mask & exclude_features_) == 0) {
         bool allowed = allowed_names_.empty();
@@ -1023,11 +1042,15 @@ INSTANTIATE_TEST_SUITE_P(
         .Run(),
     NameFromConfig);
 
+// TODO(ctiller): Resolve the ResourceQuota / EventEngine listener problems and
+// re-enable this test.
+GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(ResourceQuotaTest);
 INSTANTIATE_TEST_SUITE_P(
     ResourceQuotaTests, ResourceQuotaTest,
     ConfigQuery()
         .ExcludeFeatures(FEATURE_MASK_SUPPORTS_REQUEST_PROXYING |
-                         FEATURE_MASK_1BYTE_AT_A_TIME)
+                         FEATURE_MASK_1BYTE_AT_A_TIME |
+                         FEATURE_MASK_DISABLE_EVENT_ENGINE_LISTENER_EXPERIMENT)
         .ExcludeName("Chttp2.*Uds.*")
         .ExcludeName("Chttp2HttpProxy")
         .Run(),

test/core/end2end/end2end_tests.cc

@@ -30,8 +30,10 @@
 
 #include "src/core/lib/compression/message_compress.h"
 #include "src/core/lib/config/core_configuration.h"
+#include "src/core/lib/event_engine/default_event_engine.h"
 #include "src/core/lib/gprpp/no_destruct.h"
 #include "test/core/end2end/cq_verifier.h"
+#include "test/core/event_engine/event_engine_test_utils.h"
 
 namespace grpc_core {
 
@@ -83,6 +85,15 @@ void CoreEnd2endTest::TearDown() {
   const bool do_shutdown = fixture_ != nullptr;
   cq_verifier_.reset();
   fixture_.reset();
+// TODO(hork): locate the windows leak so we can enable end2end experiments.
+#ifndef GPR_WINDOWS
+  // Creating an EventEngine requires gRPC initialization, which the NoOp test
+  // does not do. Skip the EventEngine check if unnecessary.
+  if (grpc_is_initialized()) {
+    grpc_event_engine::experimental::WaitForSingleOwner(
+        grpc_event_engine::experimental::GetDefaultEventEngine());
+  }
+#endif
   if (do_shutdown) {
     grpc_shutdown_blocking();
     // This will wait until gRPC shutdown has actually happened to make sure

test/core/end2end/end2end_tests.h

@@ -58,23 +58,25 @@
 #include "test/core/util/test_config.h"
 
 // Test feature flags.
-#define FEATURE_MASK_DOES_NOT_SUPPORT_RETRY 1
+#define FEATURE_MASK_DOES_NOT_SUPPORT_RETRY 1 << 0
-#define FEATURE_MASK_SUPPORTS_HOSTNAME_VERIFICATION 2
+#define FEATURE_MASK_SUPPORTS_HOSTNAME_VERIFICATION 1 << 1
 // Feature mask supports call credentials with a minimum security level of
 // GRPC_PRIVACY_AND_INTEGRITY.
-#define FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS 4
+#define FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS 1 << 2
 // Feature mask supports call credentials with a minimum security level of
 // GRPC_SECURTITY_NONE.
-#define FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS_LEVEL_INSECURE 8
+#define FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS_LEVEL_INSECURE 1 << 3
-#define FEATURE_MASK_SUPPORTS_REQUEST_PROXYING 16
+#define FEATURE_MASK_SUPPORTS_REQUEST_PROXYING 1 << 4
-#define FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL 32
+#define FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL 1 << 5
-#define FEATURE_MASK_IS_HTTP2 64
+#define FEATURE_MASK_IS_HTTP2 1 << 6
-#define FEATURE_MASK_ENABLES_TRACES 128
+#define FEATURE_MASK_ENABLES_TRACES 1 << 7
-#define FEATURE_MASK_1BYTE_AT_A_TIME 256
+#define FEATURE_MASK_1BYTE_AT_A_TIME 1 << 8
-#define FEATURE_MASK_DOES_NOT_SUPPORT_WRITE_BUFFERING 512
+#define FEATURE_MASK_DOES_NOT_SUPPORT_WRITE_BUFFERING 1 << 9
-#define FEATURE_MASK_DOES_NOT_SUPPORT_CLIENT_HANDSHAKE_COMPLETE_FIRST 1024
+#define FEATURE_MASK_DOES_NOT_SUPPORT_CLIENT_HANDSHAKE_COMPLETE_FIRST 1 << 10
-#define FEATURE_MASK_IS_MINSTACK 2048
+#define FEATURE_MASK_IS_MINSTACK 1 << 11
-#define FEATURE_MASK_IS_SECURE 4096
+#define FEATURE_MASK_IS_SECURE 1 << 12
+#define FEATURE_MASK_DISABLE_EVENT_ENGINE_CLIENT_EXPERIMENT 1 << 13
+#define FEATURE_MASK_DISABLE_EVENT_ENGINE_LISTENER_EXPERIMENT 1 << 14
 
 #define FAIL_AUTH_CHECK_SERVER_ARG_NAME "fail_auth_check"