Chiebot-Mirror
/
grpc
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[build] Exclude some e2e suites from experiments tests (#34404)

Browse Source 
We have a bunch of experiments testing against core e2e - and this is
good for robustness, bad for CI times.

We also have a bunch of marginal but overall necessary fixtures in the
e2e suites - again good for robustness, bad for CI times.

We can eliminate some of the cross product though, and I think safely:
run experiments on a broad range of suites, but not *ALL* the suites,
and get a bunch of our CI time back.

Here I introduce an environment variable: `GRPC_CI_EXPERIMENTS` that's
set when running bazel @experiment= configs, cleared otherwise (so we
can still execute those tests directly when necessary). When that env
var is set we filter out a bunch of suites from the test configurations.
pull/34409/head
Craig Tiller 2 years ago committed by GitHub
parent 62521a889f
commit accc1688a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 51 additions and 22 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      bazel/grpc_build_system.bzl
  2. 1
      test/core/end2end/BUILD
  3. 68
      test/core/end2end/end2end_test_suites.cc
  4. 2
      test/core/end2end/end2end_tests.h

2
bazel/grpc_build_system.bzl
Unescape View File

@ -426,6 +426,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
config["name"] = config["name"] + "@experiment=" + experiment config["name"] = config["name"] + "@experiment=" + experiment
env = dict(config["env"]) env = dict(config["env"])
env["GRPC_EXPERIMENTS"] = experiment env["GRPC_EXPERIMENTS"] = experiment
env["GRPC_CI_EXPERIMENTS"] = "1"
config["env"] = env config["env"] = env
tags = config["tags"] tags = config["tags"]
for tag in must_have_tags + enabled_tags: for tag in must_have_tags + enabled_tags:
@ -441,6 +442,7 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
config["name"] = config["name"] + "@experiment=no_" + experiment config["name"] = config["name"] + "@experiment=no_" + experiment
env = dict(config["env"]) env = dict(config["env"])
env["GRPC_EXPERIMENTS"] = "-" + experiment env["GRPC_EXPERIMENTS"] = "-" + experiment
env["GRPC_CI_EXPERIMENTS"] = "1"
config["env"] = env config["env"] = env
tags = config["tags"] tags = config["tags"]
for tag in must_have_tags + disabled_tags: for tag in must_have_tags + disabled_tags:

1
test/core/end2end/BUILD
Unescape View File

@ -213,6 +213,7 @@ grpc_cc_library(
"//:grpc_public_hdrs", "//:grpc_public_hdrs",
"//:grpc_trace", "//:grpc_trace",
"//src/core:channel_args", "//src/core:channel_args",
"//src/core:env",
"//src/core:error", "//src/core:error",
"//src/core:grpc_fake_credentials", "//src/core:grpc_fake_credentials",
"//src/core:iomgr_port", "//src/core:iomgr_port",

68
test/core/end2end/end2end_test_suites.cc
Unescape View File

@ -46,6 +46,7 @@
#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/debug/trace.h" #include "src/core/lib/debug/trace.h"
#include "src/core/lib/gprpp/env.h"
#include "src/core/lib/gprpp/host_port.h" #include "src/core/lib/gprpp/host_port.h"
#include "src/core/lib/gprpp/no_destruct.h" #include "src/core/lib/gprpp/no_destruct.h"
#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/gprpp/sync.h"
@ -539,11 +540,13 @@ class InsecureFixtureWithPipeForWakeupFd : public InsecureFixture {
std::vector<CoreTestConfiguration> AllConfigs() { std::vector<CoreTestConfiguration> AllConfigs() {
std::vector<CoreTestConfiguration> configs { std::vector<CoreTestConfiguration> configs {
#ifdef GRPC_POSIX_SOCKET #ifdef GRPC_POSIX_SOCKET
CoreTestConfiguration{ CoreTestConfiguration{"Chttp2Fd",
"Chttp2Fd", FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, nullptr, FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ |
[](const ChannelArgs&, const ChannelArgs&) { FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
return std::make_unique<FdFixture>(); nullptr,
}}, [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<FdFixture>();
}},
#endif #endif
CoreTestConfiguration{ CoreTestConfiguration{
"Chttp2FakeSecurityFullstack", "Chttp2FakeSecurityFullstack",
@ -574,7 +577,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
"Chttp2FullstackLocalAbstractUdsPercentEncoded", "Chttp2FullstackLocalAbstractUdsPercentEncoded",
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL |
FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS |
FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr, nullptr,
[](const ChannelArgs& /*client_args*/, [](const ChannelArgs& /*client_args*/,
const ChannelArgs& /*server_args*/) { const ChannelArgs& /*server_args*/) {
@ -592,7 +596,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL |
FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS |
FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_IS_HTTP2 |
FEATURE_MASK_DO_NOT_FUZZ, FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr, nullptr,
[](const ChannelArgs& /*client_args*/, [](const ChannelArgs& /*client_args*/,
const ChannelArgs& /*server_args*/) { const ChannelArgs& /*server_args*/) {
@ -604,7 +609,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL |
FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS |
FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_IS_HTTP2 |
FEATURE_MASK_DO_NOT_FUZZ, FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr, nullptr,
[](const ChannelArgs& /*client_args*/, [](const ChannelArgs& /*client_args*/,
const ChannelArgs& /*server_args*/) { const ChannelArgs& /*server_args*/) {
@ -617,7 +623,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
"Chttp2FullstackLocalUdsPercentEncoded", "Chttp2FullstackLocalUdsPercentEncoded",
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL |
FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS |
FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr, nullptr,
[](const ChannelArgs& /*client_args*/, [](const ChannelArgs& /*client_args*/,
const ChannelArgs& /*server_args*/) { const ChannelArgs& /*server_args*/) {
@ -634,7 +641,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
"Chttp2FullstackLocalUds", "Chttp2FullstackLocalUds",
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL |
FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS |
FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr, nullptr,
[](const ChannelArgs& /*client_args*/, [](const ChannelArgs& /*client_args*/,
const ChannelArgs& /*server_args*/) { const ChannelArgs& /*server_args*/) {
@ -696,7 +704,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
"Chttp2InsecureCredentials", "Chttp2InsecureCredentials",
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL |
FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS_LEVEL_INSECURE | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS_LEVEL_INSECURE |
FEATURE_MASK_IS_HTTP2, FEATURE_MASK_IS_HTTP2 |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr, nullptr,
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<InsecureCredsFixture>(); return std::make_unique<InsecureCredsFixture>();
@ -706,7 +715,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
"Chttp2SimpleSslWithOauth2FullstackTls12", "Chttp2SimpleSslWithOauth2FullstackTls12",
FEATURE_MASK_IS_SECURE | FEATURE_MASK_IS_SECURE |
FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS |
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2, FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
"foo.test.google.fr", "foo.test.google.fr",
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<Oauth2Fixture>(grpc_tls_version::TLS1_2); return std::make_unique<Oauth2Fixture>(grpc_tls_version::TLS1_2);
@ -724,7 +734,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
"Chttp2SimplSslFullstackTls12", "Chttp2SimplSslFullstackTls12",
FEATURE_MASK_IS_SECURE | FEATURE_MASK_IS_SECURE |
FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS |
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2, FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
"foo.test.google.fr", "foo.test.google.fr",
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<SslTlsFixture>(grpc_tls_version::TLS1_2); return std::make_unique<SslTlsFixture>(grpc_tls_version::TLS1_2);
@ -742,14 +753,17 @@ std::vector<CoreTestConfiguration> AllConfigs() {
}}, }},
CoreTestConfiguration{ CoreTestConfiguration{
"Chttp2SocketPair", "Chttp2SocketPair",
FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ, nullptr, FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr,
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<SockpairFixture>(ChannelArgs()); return std::make_unique<SockpairFixture>(ChannelArgs());
}}, }},
CoreTestConfiguration{ CoreTestConfiguration{
"Chttp2SocketPair1ByteAtATime", "Chttp2SocketPair1ByteAtATime",
FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_1BYTE_AT_A_TIME | FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_1BYTE_AT_A_TIME |
FEATURE_MASK_DO_NOT_FUZZ, FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr, nullptr,
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<SockpairFixture>( return std::make_unique<SockpairFixture>(
@ -779,7 +793,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
"Chttp2SslCredReloadTls12", "Chttp2SslCredReloadTls12",
FEATURE_MASK_IS_SECURE | FEATURE_MASK_IS_SECURE |
FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS | FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS |
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2, FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
"foo.test.google.fr", "foo.test.google.fr",
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<SslCredReloadFixture>(TLS1_2); return std::make_unique<SslCredReloadFixture>(TLS1_2);
@ -799,7 +814,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
// server: certificate watcher provider + async external verifier // server: certificate watcher provider + async external verifier
// extra: TLS 1.3 // extra: TLS 1.3
"Chttp2CertWatcherProviderAsyncVerifierTls13", "Chttp2CertWatcherProviderAsyncVerifierTls13",
kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ, kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
"foo.test.google.fr", "foo.test.google.fr",
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<TlsFixture>( return std::make_unique<TlsFixture>(
@ -813,7 +829,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
// server: certificate watcher provider + sync external verifier // server: certificate watcher provider + sync external verifier
// extra: TLS 1.2 // extra: TLS 1.2
"Chttp2CertWatcherProviderSyncVerifierTls12", "Chttp2CertWatcherProviderSyncVerifierTls12",
kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ, kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
"foo.test.google.fr", "foo.test.google.fr",
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<TlsFixture>( return std::make_unique<TlsFixture>(
@ -841,7 +858,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
// server: static data provider + async external verifier // server: static data provider + async external verifier
// extra: TLS 1.3 // extra: TLS 1.3
"Chttp2StaticProviderAsyncVerifierTls13", "Chttp2StaticProviderAsyncVerifierTls13",
kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ, kH2TLSFeatureMask | FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
"foo.test.google.fr", "foo.test.google.fr",
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
return std::make_unique<TlsFixture>( return std::make_unique<TlsFixture>(
@ -854,7 +872,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
CoreTestConfiguration{ CoreTestConfiguration{
"Chttp2FullstackUdsAbstractNamespace", "Chttp2FullstackUdsAbstractNamespace",
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 |
FEATURE_MASK_DO_NOT_FUZZ, FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr, nullptr,
[](const ChannelArgs&, const ChannelArgs&) { [](const ChannelArgs&, const ChannelArgs&) {
gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
@ -908,7 +927,8 @@ std::vector<CoreTestConfiguration> AllConfigs() {
CoreTestConfiguration{ CoreTestConfiguration{
"Chttp2FullstackWithPipeWakeup", "Chttp2FullstackWithPipeWakeup",
FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 | FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL | FEATURE_MASK_IS_HTTP2 |
FEATURE_MASK_DO_NOT_FUZZ, FEATURE_MASK_DO_NOT_FUZZ |
FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS,
nullptr, nullptr,
[](const ChannelArgs& /*client_args*/, [](const ChannelArgs& /*client_args*/,
const ChannelArgs& /*server_args*/) { const ChannelArgs& /*server_args*/) {
@ -927,7 +947,11 @@ std::vector<CoreTestConfiguration> AllConfigs() {
// that match some criteria. // that match some criteria.
class ConfigQuery { class ConfigQuery {
public: public:
ConfigQuery() = default; ConfigQuery() {
if (GetEnv("GRPC_CI_EXPERIMENTS").has_value()) {
exclude_features_ |= FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS;
}
}
ConfigQuery(const ConfigQuery&) = delete; ConfigQuery(const ConfigQuery&) = delete;
ConfigQuery& operator=(const ConfigQuery&) = delete; ConfigQuery& operator=(const ConfigQuery&) = delete;
// Enforce that the returned configurations have the given features. // Enforce that the returned configurations have the given features.

2
test/core/end2end/end2end_tests.h
Unescape View File

@ -82,6 +82,8 @@
#define FEATURE_MASK_IS_MINSTACK (1 << 11) #define FEATURE_MASK_IS_MINSTACK (1 << 11)
#define FEATURE_MASK_IS_SECURE (1 << 12) #define FEATURE_MASK_IS_SECURE (1 << 12)
#define FEATURE_MASK_DO_NOT_FUZZ (1 << 13) #define FEATURE_MASK_DO_NOT_FUZZ (1 << 13)
// Exclude this fixture from experiment runs
#define FEATURE_MASK_EXCLUDE_FROM_EXPERIMENT_RUNS (1 << 14)
#define FAIL_AUTH_CHECK_SERVER_ARG_NAME "fail_auth_check" #define FAIL_AUTH_CHECK_SERVER_ARG_NAME "fail_auth_check"

Write Preview
Loading…
Cancel
Save