From accc1688a8fbb98eac5dfaf1988ed4a8baca52ad Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 19 Sep 2023 10:20:28 -0700 Subject: [PATCH] [build] Exclude some e2e suites from experiments tests (#34404) We have a bunch of experiments testing against core e2e - and this is good for robustness, bad for CI times. We also have a bunch of marginal but overall necessary fixtures in the e2e suites - again good for robustness, bad for CI times. We can eliminate some of the cross product though, and I think safely: run experiments on a broad range of suites, but not *ALL* the suites, and get a bunch of our CI time back. Here I introduce an environment variable: `GRPC_CI_EXPERIMENTS` that's set when running bazel @experiment= configs, cleared otherwise (so we can still execute those tests directly when necessary). When that env var is set we filter out a bunch of suites from the test configurations. --- bazel/grpc_build_system.bzl | 2 + test/core/end2end/BUILD | 1 + test/core/end2end/end2end_test_suites.cc | 68 ++++++++++++++++-------- test/core/end2end/end2end_tests.h | 2 + 4 files changed, 51 insertions(+), 22 deletions(-) diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index 7d3a9e25600..49fd039b51f 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -426,6 +426,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us config["name"] = config["name"] + "@experiment=" + experiment env = dict(config["env"]) env["GRPC_EXPERIMENTS"] = experiment + env["GRPC_CI_EXPERIMENTS"] = "1" config["env"] = env tags = config["tags"] for tag in must_have_tags + enabled_tags: @@ -441,6 +442,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us config["name"] = config["name"] + "@experiment=no_" + experiment env = dict(config["env"]) env["GRPC_EXPERIMENTS"] = "-" + experiment + env["GRPC_CI_EXPERIMENTS"] = "1" config["env"] = env tags = config["tags"] for tag in must_have_tags + disabled_tags: diff --git a/test/core/end2end/BUILD b/test/core/end2end/BUILD index 62d28a5065a..35b9351f21b 100644 --- a/test/core/end2end/BUILD +++ b/test/core/end2end/BUILD @@ -213,6 +213,7 @@ grpc_cc_library( "//:grpc_public_hdrs", "//:grpc_trace", "//src/core:channel_args", + "//src/core:env", "//src/core:error", "//src/core:grpc_fake_credentials", "//src/core:iomgr_port", diff --git a/test/core/end2end/end2end_test_suites.cc b/test/core/end2end/end2end_test_suites.cc index 76b0644657e..a2d2b44c418 100644 --- a/test/core/end2end/end2end_test_suites.cc +++ b/test/core/end2end/end2end_test_suites.cc @@ -46,6 +46,7 @@ #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/env.h" #include "src/core/lib/gprpp/host_port.h" #include "src/core/lib/gprpp/no_destruct.h" #include "src/core/lib/gprpp/sync.h" @@ -539,11 +540,13 @@ class InsecureFixtureWithPipeForWakeupFd : public InsecureFixture { std::vector AllConfigs() { std::vector configs { #ifdef GRPC_POSIX_SOCKET - CoreTestConfiguration{ - "Chttp2Fd", FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, nullptr, - [](const ChannelArgs&, const ChannelArgs&) { - return std::make_unique(); - }}, + CoreTestConfiguration{"Chttp2Fd", + FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, + nullptr, + [](const ChannelArgs&, const ChannelArgs&) { + return std::make_unique(); + }}, #endif CoreTestConfiguration{ "Chttp2FakeSecurityFullstack", @@ -574,7 +577,8 @@ std::vector AllConfigs() { "Chttp2FullstackLocalAbstractUdsPercentEncoded", FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | - FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, + FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, nullptr, [](const ChannelArgs& /*client_args*/, const ChannelArgs& /*server_args*/) { @@ -592,7 +596,8 @@ std::vector AllConfigs() { FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_IS_HTTP2 | - FEATURE_MASK_DO_NOT_FUZZ, + FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, nullptr, [](const ChannelArgs& /*client_args*/, const ChannelArgs& /*server_args*/) { @@ -604,7 +609,8 @@ std::vector AllConfigs() { FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_IS_HTTP2 | - FEATURE_MASK_DO_NOT_FUZZ, + FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, nullptr, [](const ChannelArgs& /*client_args*/, const ChannelArgs& /*server_args*/) { @@ -617,7 +623,8 @@ std::vector AllConfigs() { "Chttp2FullstackLocalUdsPercentEncoded", FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | - FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, + FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, nullptr, [](const ChannelArgs& /*client_args*/, const ChannelArgs& /*server_args*/) { @@ -634,7 +641,8 @@ std::vector AllConfigs() { "Chttp2FullstackLocalUds", FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | - FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, + FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, nullptr, [](const ChannelArgs& /*client_args*/, const ChannelArgs& /*server_args*/) { @@ -696,7 +704,8 @@ std::vector AllConfigs() { "Chttp2InsecureCredentials", FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS_LEVEL_INSECURE | - FEATURE_MASK_IS_HTTP2, + FEATURE_MASK_IS_HTTP2 | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, nullptr, [](const ChannelArgs&, const ChannelArgs&) { return std::make_unique(); @@ -706,7 +715,8 @@ std::vector AllConfigs() { "Chttp2SimpleSslWithOauth2FullstackTls12", FEATURE_MASK_IS_SECURE | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | - FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2, + FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, "foo.test.google.fr", [](const ChannelArgs&, const ChannelArgs&) { return std::make_unique(grpc_tls_version::TLS1_2); @@ -724,7 +734,8 @@ std::vector AllConfigs() { "Chttp2SimplSslFullstackTls12", FEATURE_MASK_IS_SECURE | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | - FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2, + FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, "foo.test.google.fr", [](const ChannelArgs&, const ChannelArgs&) { return std::make_unique(grpc_tls_version::TLS1_2); @@ -742,14 +753,17 @@ std::vector AllConfigs() { }}, CoreTestConfiguration{ "Chttp2SocketPair", - FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, nullptr, + FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, + nullptr, [](const ChannelArgs&, const ChannelArgs&) { return std::make_unique(ChannelArgs()); }}, CoreTestConfiguration{ "Chttp2SocketPair1ByteAtATime", FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_1BYTE_AT_A_TIME | - FEATURE_MASK_DO_NOT_FUZZ, + FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, nullptr, [](const ChannelArgs&, const ChannelArgs&) { return std::make_unique( @@ -779,7 +793,8 @@ std::vector AllConfigs() { "Chttp2SslCredReloadTls12", FEATURE_MASK_IS_SECURE | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | - FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2, + FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, "foo.test.google.fr", [](const ChannelArgs&, const ChannelArgs&) { return std::make_unique(TLS1_2); @@ -799,7 +814,8 @@ std::vector AllConfigs() { // server: certificate watcher provider + async external verifier // extra: TLS 1.3 "Chttp2CertWatcherProviderAsyncVerifierTls13", - kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ, + kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, "foo.test.google.fr", [](const ChannelArgs&, const ChannelArgs&) { return std::make_unique( @@ -813,7 +829,8 @@ std::vector AllConfigs() { // server: certificate watcher provider + sync external verifier // extra: TLS 1.2 "Chttp2CertWatcherProviderSyncVerifierTls12", - kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ, + kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, "foo.test.google.fr", [](const ChannelArgs&, const ChannelArgs&) { return std::make_unique( @@ -841,7 +858,8 @@ std::vector AllConfigs() { // server: static data provider + async external verifier // extra: TLS 1.3 "Chttp2StaticProviderAsyncVerifierTls13", - kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ, + kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, "foo.test.google.fr", [](const ChannelArgs&, const ChannelArgs&) { return std::make_unique( @@ -854,7 +872,8 @@ std::vector AllConfigs() { CoreTestConfiguration{ "Chttp2FullstackUdsAbstractNamespace", FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 | - FEATURE_MASK_DO_NOT_FUZZ, + FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, nullptr, [](const ChannelArgs&, const ChannelArgs&) { gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); @@ -908,7 +927,8 @@ std::vector AllConfigs() { CoreTestConfiguration{ "Chttp2FullstackWithPipeWakeup", FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 | - FEATURE_MASK_DO_NOT_FUZZ, + FEATURE_MASK_DO_NOT_FUZZ | + FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS, nullptr, [](const ChannelArgs& /*client_args*/, const ChannelArgs& /*server_args*/) { @@ -927,7 +947,11 @@ std::vector AllConfigs() { // that match some criteria. class ConfigQuery { public: - ConfigQuery() = default; + ConfigQuery() { + if (GetEnv("GRPC_CI_EXPERIMENTS").has_value()) { + exclude_features_ |= FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS; + } + } ConfigQuery(const ConfigQuery&) = delete; ConfigQuery& operator=(const ConfigQuery&) = delete; // Enforce that the returned configurations have the given features. diff --git a/test/core/end2end/end2end_tests.h b/test/core/end2end/end2end_tests.h index a7fcb6aea45..5d8a607b0a1 100644 --- a/test/core/end2end/end2end_tests.h +++ b/test/core/end2end/end2end_tests.h @@ -82,6 +82,8 @@ #define FEATURE_MASK_IS_MINSTACK (1 << 11) #define FEATURE_MASK_IS_SECURE (1 << 12) #define FEATURE_MASK_DO_NOT_FUZZ (1 << 13) +// Exclude this fixture from experiment runs +#define FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS (1 << 14) #define FAIL_AUTH_CHECK_SERVER_ARG_NAME "fail_auth_check"