Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

end2end tests: apply test slowdown factor in various places where it was missed (#30749)

Browse Source 
* e2e tests: add test scaling factor to durations in channel args

* apply test scaling factor when encoding durations in xDS protos

* apply scaling factor in fixed timeout in WaitForNack()

* fix overflow

* clang-format

* adjust timeouts in fault injection tests

* add missing slowdown factor

* clang-format

* add tests for duration multiplication
pull/30756/head^2
Mark D. Roth 2 years ago committed by GitHub
parent 8b941bb648
commit 121a08f6a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 326 additions and 438 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      src/core/lib/gprpp/time.h
  2. 7
      test/core/gprpp/time_test.cc
  3. 32
      test/cpp/end2end/client_lb_end2end_test.cc
  4. 14
      test/cpp/end2end/xds/xds_end2end_test_lib.cc
  5. 11
      test/cpp/end2end/xds/xds_end2end_test_lib.h
  6. 99
      test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc
  7. 397
      test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc
  8. 3
      test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc
  9. 176
      test/cpp/end2end/xds/xds_routing_end2end_test.cc
  10. 3
      test/cpp/end2end/xds/xds_server.h

6
src/core/lib/gprpp/time.h
Unescape View File

@ -200,6 +200,7 @@ class Duration {
}
return *this;
}
Duration& operator*=(double multiplier);
Duration& operator+=(Duration other) {
millis_ += other.millis_;
return *this;
@ -287,6 +288,11 @@ inline Duration Duration::FromSecondsAsDouble(double seconds) {
return Milliseconds(static_cast<int64_t>(millis));
}
inline Duration& Duration::operator*=(double multiplier) {
*this = *this * multiplier;
return *this;
}
inline Timestamp& Timestamp::operator+=(Duration duration) {
return *this = (*this + duration);
}

7
test/core/gprpp/time_test.cc
Unescape View File

@ -75,6 +75,13 @@ TEST(DurationTest, Infinities) {
Duration::NegativeInfinity());
}
TEST(DurationTest, Multiplication) {
Duration d = Duration::Seconds(5);
EXPECT_EQ(d * 2, Duration::Seconds(10));
d *= 3;
EXPECT_EQ(d, Duration::Seconds(15));
}
TEST(DurationTest, FromTimespan) {
EXPECT_EQ(Duration::FromTimespec(gpr_time_from_millis(1234, GPR_TIMESPAN)),
Duration::Milliseconds(1234));

32
test/cpp/end2end/client_lb_end2end_test.cc
Unescape View File

@ -581,7 +581,8 @@ TEST_F(ClientLbEnd2endTest, ChannelIdleness) {
StartServers(kNumServers);
// Set max idle time and build the channel.
ChannelArguments args;
args.SetInt(GRPC_ARG_CLIENT_IDLE_TIMEOUT_MS, 1000);
args.SetInt(GRPC_ARG_CLIENT_IDLE_TIMEOUT_MS,
1000 * grpc_test_slowdown_factor());
auto response_generator = BuildResolverResponseGenerator();
auto channel = BuildChannel("", response_generator, args);
auto stub = BuildStub(channel);
@ -732,7 +733,8 @@ TEST_F(PickFirstTest, ProcessPending) {
TEST_F(PickFirstTest, SelectsReadyAtStartup) {
ChannelArguments args;
constexpr int kInitialBackOffMs = 5000;
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs);
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
kInitialBackOffMs * grpc_test_slowdown_factor());
// Create 2 servers, but start only the second one.
std::vector<int> ports = {grpc_pick_unused_port_or_die(),
grpc_pick_unused_port_or_die()};
@ -758,7 +760,8 @@ TEST_F(PickFirstTest, SelectsReadyAtStartup) {
TEST_F(PickFirstTest, BackOffInitialReconnect) {
ChannelArguments args;
constexpr int kInitialBackOffMs = 100;
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs);
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
kInitialBackOffMs * grpc_test_slowdown_factor());
const std::vector<int> ports = {grpc_pick_unused_port_or_die()};
const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC);
auto response_generator = BuildResolverResponseGenerator();
@ -779,7 +782,8 @@ TEST_F(PickFirstTest, BackOffInitialReconnect) {
gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited.millis());
// We should have waited at least kInitialBackOffMs. We substract one to
// account for test and precision accuracy drift.
EXPECT_GE(waited.millis(), kInitialBackOffMs - 1);
EXPECT_GE(waited.millis(),
(kInitialBackOffMs * grpc_test_slowdown_factor()) - 1);
// But not much more.
EXPECT_GT(
gpr_time_cmp(
@ -790,7 +794,8 @@ TEST_F(PickFirstTest, BackOffInitialReconnect) {
TEST_F(PickFirstTest, BackOffMinReconnect) {
ChannelArguments args;
constexpr int kMinReconnectBackOffMs = 1000;
args.SetInt(GRPC_ARG_MIN_RECONNECT_BACKOFF_MS, kMinReconnectBackOffMs);
args.SetInt(GRPC_ARG_MIN_RECONNECT_BACKOFF_MS,
kMinReconnectBackOffMs * grpc_test_slowdown_factor());
const std::vector<int> ports = {grpc_pick_unused_port_or_die()};
auto response_generator = BuildResolverResponseGenerator();
auto channel = BuildChannel("pick_first", response_generator, args);
@ -799,8 +804,8 @@ TEST_F(PickFirstTest, BackOffMinReconnect) {
// Make connection delay a 10% longer than it's willing to in order to make
// sure we are hitting the codepath that waits for the min reconnect backoff.
ConnectionAttemptInjector injector;
injector.SetDelay(
grpc_core::Duration::Milliseconds(kMinReconnectBackOffMs * 1.10));
injector.SetDelay(grpc_core::Duration::Milliseconds(
kMinReconnectBackOffMs * grpc_test_slowdown_factor() * 1.10));
const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC);
channel->WaitForConnected(
grpc_timeout_milliseconds_to_deadline(kMinReconnectBackOffMs * 2));
@ -810,13 +815,15 @@ TEST_F(PickFirstTest, BackOffMinReconnect) {
gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited.millis());
// We should have waited at least kMinReconnectBackOffMs. We substract one to
// account for test and precision accuracy drift.
EXPECT_GE(waited.millis(), kMinReconnectBackOffMs - 1);
EXPECT_GE(waited.millis(),
(kMinReconnectBackOffMs * grpc_test_slowdown_factor()) - 1);
}
TEST_F(PickFirstTest, ResetConnectionBackoff) {
ChannelArguments args;
constexpr int kInitialBackOffMs = 1000;
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs);
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
kInitialBackOffMs * grpc_test_slowdown_factor());
const std::vector<int> ports = {grpc_pick_unused_port_or_die()};
auto response_generator = BuildResolverResponseGenerator();
auto channel = BuildChannel("pick_first", response_generator, args);
@ -844,7 +851,7 @@ TEST_F(PickFirstTest, ResetConnectionBackoff) {
grpc_core::Duration::FromTimespec(gpr_time_sub(t1, t0));
gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited.millis());
// We should have waited less than kInitialBackOffMs.
EXPECT_LT(waited.millis(), kInitialBackOffMs);
EXPECT_LT(waited.millis(), kInitialBackOffMs * grpc_test_slowdown_factor());
}
TEST_F(ClientLbEnd2endTest,
@ -854,8 +861,9 @@ TEST_F(ClientLbEnd2endTest,
// Create client.
const int port = grpc_pick_unused_port_or_die();
ChannelArguments args;
const int kInitialBackOffMs = 5000 * grpc_test_slowdown_factor();
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs);
const int kInitialBackOffMs = 5000;
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
kInitialBackOffMs * grpc_test_slowdown_factor());
auto response_generator = BuildResolverResponseGenerator();
auto channel = BuildChannel("pick_first", response_generator, args);
auto stub = BuildStub(channel);

14
test/cpp/end2end/xds/xds_end2end_test_lib.cc
Unescape View File

@ -791,7 +791,8 @@ std::shared_ptr<Channel> XdsEnd2endTest::CreateChannel(
// TODO(roth): Remove this once we enable retries by default internally.
args->SetInt(GRPC_ARG_ENABLE_RETRIES, 1);
if (failover_timeout_ms > 0) {
args->SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, failover_timeout_ms);
args->SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS,
failover_timeout_ms * grpc_test_slowdown_factor());
}
if (GetParam().bootstrap_source() == XdsTestType::kBootstrapFromChannelArg) {
// We're getting the bootstrap from a channel arg, so we do the
@ -1004,7 +1005,8 @@ absl::optional<AdsServiceImpl::ResponseState> XdsEnd2endTest::WaitForNack(
std::function<absl::optional<AdsServiceImpl::ResponseState>()> get_state,
const RpcOptions& rpc_options, StatusCode expected_status) {
absl::optional<AdsServiceImpl::ResponseState> response_state;
auto deadline = absl::Now() + absl::Seconds(30);
auto deadline =
absl::Now() + (absl::Seconds(30) * grpc_test_slowdown_factor());
auto continue_predicate = [&]() {
if (absl::Now() >= deadline) {
return false;
@ -1023,6 +1025,14 @@ absl::optional<AdsServiceImpl::ResponseState> XdsEnd2endTest::WaitForNack(
return response_state;
}
void XdsEnd2endTest::SetProtoDuration(
grpc_core::Duration duration, google::protobuf::Duration* duration_proto) {
duration *= grpc_test_slowdown_factor();
gpr_timespec ts = duration.as_timespec();
duration_proto->set_seconds(ts.tv_sec);
duration_proto->set_nanos(ts.tv_nsec);
}
std::string XdsEnd2endTest::ReadFile(const char* file_path) {
grpc_slice slice;
GPR_ASSERT(

11
test/cpp/end2end/xds/xds_end2end_test_lib.h
Unescape View File

@ -720,6 +720,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
// Dev note: If a timeout or Deadline Exceeded error is occurring in an XDS
// end2end test, consider changing that test's timeout instead of this
// global default.
// Will be multiplied by grpc_test_slowdown_factor().
int timeout_ms = 1000;
bool wait_for_ready = false;
std::vector<std::pair<std::string, std::string>> metadata;
@ -747,6 +748,11 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
return *this;
}
RpcOptions& set_timeout(grpc_core::Duration timeout) {
timeout_ms = timeout.millis();
return *this;
}
RpcOptions& set_wait_for_ready(bool rpc_wait_for_ready) {
wait_for_ready = rpc_wait_for_ready;
return *this;
@ -884,6 +890,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
// If true, resets the backend counters before returning.
bool reset_counters = true;
// How long to wait for the backend(s) to see requests.
// Will be multiplied by grpc_test_slowdown_factor().
int timeout_ms = 5000;
WaitForBackendOptions() {}
@ -1012,6 +1019,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
gpr_now(GPR_CLOCK_MONOTONIC));
}
// Sets duration_proto to duration times grpc_test_slowdown_factor().
static void SetProtoDuration(grpc_core::Duration duration,
google::protobuf::Duration* duration_proto);
// Returns the number of RPCs needed to pass error_tolerance at 99.99994%
// chance. Rolling dices in drop/fault-injection generates a binomial
// distribution (if our code is not horribly wrong). Let's make "n" the number

99
test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc
Unescape View File

@ -204,8 +204,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) {
TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
CreateAndStartBackends(1);
const uint32_t kRpcTimeoutMilliseconds = 10000;
const uint32_t kFixedDelaySeconds = 100;
const auto kRpcTimeout = grpc_core::Duration::Seconds(10);
const auto kFixedDelay = grpc_core::Duration::Seconds(20);
const uint32_t kDelayPercentagePerHundred = 50;
const double kDelayRate = kDelayPercentagePerHundred / 100.0;
const double kErrorTolerance = 0.1;
@ -225,14 +225,13 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(kDelayPercentagePerHundred);
delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
fixed_delay->set_seconds(kFixedDelaySeconds);
SetProtoDuration(kFixedDelay,
http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup
SetFilterConfig(http_fault);
// Send kNumRpcs RPCs and count the delays.
RpcOptions rpc_options = RpcOptions()
.set_timeout_ms(kRpcTimeoutMilliseconds)
.set_skip_cancelled_check(true);
RpcOptions rpc_options =
RpcOptions().set_timeout(kRpcTimeout).set_skip_cancelled_check(true);
std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
size_t num_delayed = 0;
@ -249,8 +248,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
CreateAndStartBackends(1);
const uint32_t kFixedDelayMilliseconds = 100000;
const uint32_t kRpcTimeoutMilliseconds = 10000;
const auto kRpcTimeout = grpc_core::Duration::Seconds(10);
const auto kFixedDelay = grpc_core::Duration::Seconds(20);
const uint32_t kDelayPercentageCap = 100;
const uint32_t kDelayPercentage = 50;
const double kDelayRate = kDelayPercentage / 100.0;
@ -275,13 +274,14 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
SetFilterConfig(http_fault);
// Send kNumRpcs RPCs and count the delays.
std::vector<std::pair<std::string, std::string>> metadata = {
{"x-envoy-fault-delay-request", std::to_string(kFixedDelayMilliseconds)},
{"x-envoy-fault-delay-request",
std::to_string(kFixedDelay.millis() * grpc_test_slowdown_factor())},
{"x-envoy-fault-delay-request-percentage",
std::to_string(kDelayPercentage)},
};
RpcOptions rpc_options = RpcOptions()
.set_metadata(metadata)
.set_timeout_ms(kRpcTimeoutMilliseconds)
.set_timeout(kRpcTimeout)
.set_skip_cancelled_check(true);
std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
@ -299,8 +299,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) {
CreateAndStartBackends(1);
const uint32_t kFixedDelaySeconds = 1;
const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach
const auto kRpcTimeout = grpc_core::Duration::Seconds(30);
const auto kFixedDelay = grpc_core::Duration::Seconds(1);
// Create an EDS resource
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
@ -314,14 +314,14 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(100); // Always inject DELAY!
delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
fixed_delay->set_seconds(kFixedDelaySeconds);
SetProtoDuration(kFixedDelay,
http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup
SetFilterConfig(http_fault);
// Send a stream RPC and check its status code
ClientContext context;
context.set_deadline(
grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds));
grpc_timeout_milliseconds_to_deadline(kRpcTimeout.millis()));
auto stream = stub_->BidiStream(&context);
stream->WritesDone();
auto status = stream->Finish();
@ -332,12 +332,11 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) {
TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
CreateAndStartBackends(1);
const auto kConnectTimeout = grpc_core::Duration::Seconds(10);
const auto kRpcTimeout = grpc_core::Duration::Seconds(30);
const auto kFixedDelay = grpc_core::Duration::Seconds(1);
const uint32_t kAbortPercentagePerHundred = 50;
const double kAbortRate = kAbortPercentagePerHundred / 100.0;
const uint32_t kFixedDelaySeconds = 1;
const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach
const uint32_t kConnectionTimeoutMilliseconds =
10 * 1000; // 10s should not reach
const double kErrorTolerance = 0.1;
const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
const size_t kMaxConcurrentRequests = kNumRpcs;
@ -360,23 +359,22 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(1000000); // Always inject DELAY!
delay_percentage->set_denominator(FractionalPercent::MILLION);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
fixed_delay->set_seconds(kFixedDelaySeconds);
SetProtoDuration(kFixedDelay,
http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup
SetFilterConfig(http_fault);
// Allow the channel to connect to one backends, so the herd of queued RPCs
// won't be executed on the same ExecCtx object and using the cached Now()
// value, which causes millisecond level delay error.
channel_->WaitForConnected(
grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds));
grpc_timeout_milliseconds_to_deadline(kConnectTimeout.millis()));
// Send kNumRpcs RPCs and count the aborts.
int num_aborted = 0;
RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds);
RpcOptions rpc_options = RpcOptions().set_timeout(kRpcTimeout);
std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
for (auto& rpc : rpcs) {
EXPECT_GE(rpc.elapsed_time,
grpc_core::Duration::Seconds(kFixedDelaySeconds));
EXPECT_GE(rpc.elapsed_time, kFixedDelay * grpc_test_slowdown_factor());
if (rpc.status.error_code() == StatusCode::OK) continue;
EXPECT_EQ("Fault injected", rpc.status.error_message());
++num_aborted;
@ -393,12 +391,11 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
TEST_P(FaultInjectionTest,
XdsFaultInjectionAlwaysDelayPercentageAbortSwitchDenominator) {
CreateAndStartBackends(1);
const auto kConnectTimeout = grpc_core::Duration::Seconds(10);
const auto kRpcTimeout = grpc_core::Duration::Seconds(30);
const auto kFixedDelay = grpc_core::Duration::Seconds(1);
const uint32_t kAbortPercentagePerMillion = 500000;
const double kAbortRate = kAbortPercentagePerMillion / 1000000.0;
const uint32_t kFixedDelaySeconds = 1; // 1s
const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach
const uint32_t kConnectionTimeoutMilliseconds =
10 * 1000; // 10s should not reach
const double kErrorTolerance = 0.1;
const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
const size_t kMaxConcurrentRequests = kNumRpcs;
@ -421,23 +418,22 @@ TEST_P(FaultInjectionTest,
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(100); // Always inject DELAY!
delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
fixed_delay->set_seconds(kFixedDelaySeconds);
SetProtoDuration(kFixedDelay,
http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup
SetFilterConfig(http_fault);
// Allow the channel to connect to one backends, so the herd of queued RPCs
// won't be executed on the same ExecCtx object and using the cached Now()
// value, which causes millisecond level delay error.
channel_->WaitForConnected(
grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds));
grpc_timeout_milliseconds_to_deadline(kConnectTimeout.millis()));
// Send kNumRpcs RPCs and count the aborts.
int num_aborted = 0;
RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds);
RpcOptions rpc_options = RpcOptions().set_timeout(kRpcTimeout);
std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
for (auto& rpc : rpcs) {
EXPECT_GE(rpc.elapsed_time,
grpc_core::Duration::Seconds(kFixedDelaySeconds));
EXPECT_GE(rpc.elapsed_time, kFixedDelay * grpc_test_slowdown_factor());
if (rpc.status.error_code() == StatusCode::OK) continue;
EXPECT_EQ("Fault injected", rpc.status.error_message());
++num_aborted;
@ -450,10 +446,10 @@ TEST_P(FaultInjectionTest,
TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
CreateAndStartBackends(1);
const auto kRpcTimeout = grpc_core::Duration::Seconds(4);
const auto kFixedDelay = grpc_core::Duration::Seconds(20);
const uint32_t kMaxFault = 10;
const uint32_t kNumRpcs = 30; // kNumRpcs should be bigger than kMaxFault
const uint32_t kRpcTimeoutMs = 4000; // 4 seconds
const uint32_t kLongDelaySeconds = 100; // 100 seconds
const uint32_t kAlwaysDelayPercentage = 100;
// Create an EDS resource
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
@ -464,15 +460,15 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
delay_percentage->set_numerator(
kAlwaysDelayPercentage); // Always inject DELAY!
delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
fixed_delay->set_seconds(kLongDelaySeconds);
SetProtoDuration(kFixedDelay,
http_fault.mutable_delay()->mutable_fixed_delay());
http_fault.mutable_max_active_faults()->set_value(kMaxFault);
// Config fault injection via different setup
SetFilterConfig(http_fault);
// Sends a batch of long running RPCs with long timeout to consume all
// active faults quota.
int num_delayed = 0;
RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMs);
RpcOptions rpc_options = RpcOptions().set_timeout(kRpcTimeout);
std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
for (auto& rpc : rpcs) {
@ -498,9 +494,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) {
CreateAndStartBackends(1);
// kRpcTimeoutMilliseconds is 10s should never be reached.
const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 10000;
const uint32_t kFixedDelaySeconds = 1;
const auto kRpcTimeout = grpc_core::Duration::Seconds(20);
const auto kFixedDelay = grpc_core::Duration::Seconds(1);
const uint32_t kDelayPercentagePerHundred = 100;
// Create an EDS resource
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
@ -510,13 +505,13 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(kDelayPercentagePerHundred);
delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
fixed_delay->set_seconds(kFixedDelaySeconds);
SetProtoDuration(kFixedDelay,
http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup
SetFilterConfig(http_fault);
ClientContext context;
context.set_deadline(
grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds));
grpc_timeout_milliseconds_to_deadline(kRpcTimeout.millis()));
auto stream = stub_->BidiStream(&context);
stream->WritesDone();
auto status = stream->Finish();
@ -530,8 +525,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) {
// for description.
TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayError) {
CreateAndStartBackends(1);
const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 500;
const uint32_t kFixedDelaySeconds = 100;
const auto kRpcTimeout = grpc_core::Duration::Seconds(10);
const auto kFixedDelay = grpc_core::Duration::Seconds(30);
const uint32_t kDelayPercentagePerHundred = 100;
// Create an EDS resource
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
@ -541,13 +536,13 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayError) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(kDelayPercentagePerHundred);
delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
fixed_delay->set_seconds(kFixedDelaySeconds);
SetProtoDuration(kFixedDelay,
http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup
SetFilterConfig(http_fault);
ClientContext context;
context.set_deadline(
grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds));
grpc_timeout_milliseconds_to_deadline(kRpcTimeout.millis()));
auto stream = stub_->BidiStream(&context);
stream->WritesDone();
auto status = stream->Finish();

397
test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc
Unescape View File

@ -63,23 +63,15 @@ TEST_P(OutlierDetectionTest, SuccessRateEjectionAndUnejection) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
auto* base_time =
cluster.mutable_outlier_detection()->mutable_base_ejection_time();
interval->set_seconds(1 * grpc_test_slowdown_factor());
base_time->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_success_rate_stdev_factor()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_enforcing_success_rate()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_base_ejection_time());
outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
outlier_detection->mutable_enforcing_success_rate()->set_value(100);
outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
outlier_detection->mutable_success_rate_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -135,20 +127,13 @@ TEST_P(OutlierDetectionTest, SuccessRateMaxPercent) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_success_rate_stdev_factor()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_enforcing_success_rate()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
outlier_detection->mutable_enforcing_success_rate()->set_value(100);
outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
outlier_detection->mutable_success_rate_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -239,29 +224,21 @@ TEST_P(OutlierDetectionTest, SuccessRateStdevFactor) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
auto* base_time =
cluster.mutable_outlier_detection()->mutable_base_ejection_time();
interval->set_seconds(1 * grpc_test_slowdown_factor());
base_time->set_seconds(1 * grpc_test_slowdown_factor());
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_base_ejection_time());
// We know a stdev factor of 100 will ensure the ejection occurs, so setting
// it to something higher like 1000 to test that ejection will not occur.
// Note this parameter is the only difference between this test and
// SuccessRateEjectionAndUnejection (ejection portion, value set to 100) and
// this one value changes means the difference between not ejecting in this
// test and ejecting in the other test.
cluster.mutable_outlier_detection()
->mutable_success_rate_stdev_factor()
->set_value(1000);
cluster.mutable_outlier_detection()
->mutable_enforcing_success_rate()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
outlier_detection->mutable_success_rate_stdev_factor()->set_value(1000);
outlier_detection->mutable_enforcing_success_rate()->set_value(100);
outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
outlier_detection->mutable_success_rate_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -310,28 +287,20 @@ TEST_P(OutlierDetectionTest, SuccessRateEnforcementPercentage) {
CreateAndStartBackends(2);
auto cluster = default_cluster_;
cluster.set_lb_policy(Cluster::RING_HASH);
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
auto* base_time =
cluster.mutable_outlier_detection()->mutable_base_ejection_time();
interval->set_seconds(1 * grpc_test_slowdown_factor());
base_time->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_success_rate_stdev_factor()
->set_value(100);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_base_ejection_time());
outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
// Setting enforcing_success_rate to 0 to ensure we will never eject.
// Note this parameter is the only difference between this test and
// SuccessRateEjectionAndUnejection (ejection portion, value set to 100) and
// this one value changes means the difference between guaranteed not ejecting
// in this test and guaranteed ejecting in the other test.
cluster.mutable_outlier_detection()
->mutable_enforcing_success_rate()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
outlier_detection->mutable_enforcing_success_rate()->set_value(0);
outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
outlier_detection->mutable_success_rate_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -382,25 +351,18 @@ TEST_P(OutlierDetectionTest, SuccessRateMinimumHosts) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_success_rate_stdev_factor()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_enforcing_success_rate()
->set_value(100);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
outlier_detection->mutable_enforcing_success_rate()->set_value(100);
// Set success_rate_minimum_hosts to 3 when we only have 2 backends
// Note this parameter is the only difference between this test and
// SuccessRateEjectionAndUnejection (ejection portion, value set to 1) and
// this one value changes means the difference between not ejecting in this
// test and ejecting in the other test.
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(3);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
outlier_detection->mutable_success_rate_minimum_hosts()->set_value(3);
outlier_detection->mutable_success_rate_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -451,26 +413,19 @@ TEST_P(OutlierDetectionTest, SuccessRateRequestVolume) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_success_rate_stdev_factor()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_enforcing_success_rate()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
outlier_detection->mutable_enforcing_success_rate()->set_value(100);
outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
// Set success_rate_request_volume to 4 when we only send 3 RPC in the
// interval.
// Note this parameter is the only difference between this test and
// SuccessRateEjectionAndUnejection (ejection portion, value set to 1) and
// this one value changes means the difference between not ejecting in this
// test and ejecting in the other test.
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(4);
outlier_detection->mutable_success_rate_request_volume()->set_value(4);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -527,23 +482,15 @@ TEST_P(OutlierDetectionTest, FailurePercentageEjectionAndUnejection) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
auto* base_time =
cluster.mutable_outlier_detection()->mutable_base_ejection_time();
base_time->set_seconds(3 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
SetProtoDuration(grpc_core::Duration::Seconds(3),
outlier_detection->mutable_base_ejection_time());
outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -606,20 +553,13 @@ TEST_P(OutlierDetectionTest, FailurePercentageMaxPercentage) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -707,28 +647,20 @@ TEST_P(OutlierDetectionTest, FailurePercentageThreshold) {
CreateAndStartBackends(2);
auto cluster = default_cluster_;
cluster.set_lb_policy(Cluster::RING_HASH);
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
auto* base_time =
cluster.mutable_outlier_detection()->mutable_base_ejection_time();
base_time->set_seconds(1 * grpc_test_slowdown_factor());
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_base_ejection_time());
// Setup outlier failure percentage parameter to 50
// Note this parameter is the only difference between this test and
// FailurePercentageEjectionAndUnejection (ejection portion, value set to 0)
// and this one value changes means the difference between not ejecting in
// this test and ejecting in the other test.
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(50);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
outlier_detection->mutable_failure_percentage_threshold()->set_value(50);
outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -778,28 +710,20 @@ TEST_P(OutlierDetectionTest, FailurePercentageEnforcementPercentage) {
CreateAndStartBackends(2);
auto cluster = default_cluster_;
cluster.set_lb_policy(Cluster::RING_HASH);
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
auto* base_time =
cluster.mutable_outlier_detection()->mutable_base_ejection_time();
base_time->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_base_ejection_time());
outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
// Setting enforcing_success_rate to 0 to ensure we will never eject.
// Note this parameter is the only difference between this test and
// FailurePercentageEjectionAndUnejection (ejection portion, value set to 100)
// and this one value changes means the difference between guaranteed not
// ejecting in this test and guaranteed ejecting in the other test.
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
outlier_detection->mutable_enforcing_failure_percentage()->set_value(0);
outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -851,14 +775,11 @@ TEST_P(OutlierDetectionTest, FailurePercentageMinimumHosts) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
// Set failure_percentage_minimum_hosts to 3 when we only have 2 backends
// Note this parameter is the only difference between this test and
// FailurePercentageEjectionAndUnejection (ejection portion, value set to 1)
@ -922,26 +843,19 @@ TEST_P(OutlierDetectionTest, FailurePercentageRequestVolume) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
// Set failure_percentage_request_volume to 4 when we only send 3 RPC in the
// interval.
// // Note this parameter is the only difference between this test and
// FailurePercentageEjectionAndUnejection (ejection portion, value set to 1)
// and this one value changes means the difference between not ejecting in
// this test and ejecting in the other test.
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(4);
outlier_detection->mutable_failure_percentage_request_volume()->set_value(4);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -995,37 +909,20 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_max_ejection_percent()
->set_value(50);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
outlier_detection->mutable_max_ejection_percent()->set_value(50);
// This stdev of 500 will ensure the number of ok RPC and error RPC we send
// will make 1 outlier out of the 4 backends.
cluster.mutable_outlier_detection()
->mutable_success_rate_stdev_factor()
->set_value(500);
cluster.mutable_outlier_detection()
->mutable_enforcing_success_rate()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
outlier_detection->mutable_success_rate_stdev_factor()->set_value(500);
outlier_detection->mutable_enforcing_success_rate()->set_value(100);
outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
outlier_detection->mutable_success_rate_request_volume()->set_value(1);
outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -1120,7 +1017,7 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) {
}
// Tests SuccessRate and FailurePercentage both unconfigured;
// This is the case where according to the RFC we need to instruct the picker
// This is the case where according to the gRFC we need to instruct the picker
// not to do counting or even start the timer. The result of not counting is
// that there will be no ejection taking place since we can't do any
// calculations.
@ -1130,14 +1027,11 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentageBothDisabled) {
CreateAndStartBackends(2);
auto cluster = default_cluster_;
cluster.set_lb_policy(Cluster::RING_HASH);
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
auto* base_time =
cluster.mutable_outlier_detection()->mutable_base_ejection_time();
interval->set_seconds(1 * grpc_test_slowdown_factor());
base_time->set_seconds(1 * grpc_test_slowdown_factor());
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_base_ejection_time());
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -1186,37 +1080,20 @@ TEST_P(OutlierDetectionTest,
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_max_ejection_percent()
->set_value(50);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
outlier_detection->mutable_max_ejection_percent()->set_value(50);
// This stdev of 500 will ensure the number of ok RPC and error RPC we send
// will make 1 outlier out of the 4 backends.
cluster.mutable_outlier_detection()
->mutable_success_rate_stdev_factor()
->set_value(500);
cluster.mutable_outlier_detection()
->mutable_enforcing_success_rate()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
outlier_detection->mutable_success_rate_stdev_factor()->set_value(500);
outlier_detection->mutable_enforcing_success_rate()->set_value(100);
outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
outlier_detection->mutable_success_rate_request_volume()->set_value(1);
outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -1293,23 +1170,15 @@ TEST_P(OutlierDetectionTest, DisableOutlierDetectionWhileAddressesAreEjected) {
// Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval();
interval->set_seconds(1 * grpc_test_slowdown_factor());
auto* base_time =
cluster.mutable_outlier_detection()->mutable_base_ejection_time();
base_time->set_seconds(3 * grpc_test_slowdown_factor());
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
auto* outlier_detection = cluster.mutable_outlier_detection();
SetProtoDuration(grpc_core::Duration::Seconds(1),
outlier_detection->mutable_interval());
SetProtoDuration(grpc_core::Duration::Seconds(3),
outlier_detection->mutable_base_ejection_time());
outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);

3
test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc
Unescape View File

@ -285,7 +285,8 @@ TEST_P(RingHashTest,
// Increase subchannel backoff time, so that subchannels stay in
// TRANSIENT_FAILURE for long enough to trigger potential problems.
ChannelArguments channel_args;
channel_args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, 10000);
channel_args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
10000 * grpc_test_slowdown_factor());
SetUpChannel(&channel_args);
// Start an RPC in the background.
LongRunningRpc rpc;

176
test/cpp/end2end/xds/xds_routing_end2end_test.cc
Unescape View File

@ -1882,12 +1882,12 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
}
TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
const int64_t kTimeoutMillis = 500;
const int64_t kTimeoutNano = kTimeoutMillis * 1000000;
const int64_t kTimeoutGrpcTimeoutHeaderMaxSecond = 1;
const int64_t kTimeoutMaxStreamDurationSecond = 2;
const int64_t kTimeoutHttpMaxStreamDurationSecond = 3;
const int64_t kTimeoutApplicationSecond = 4;
const auto kTimeoutGrpcHeaderMax = grpc_core::Duration::Milliseconds(1500);
const auto kTimeoutMaxStreamDuration =
grpc_core::Duration::Milliseconds(2500);
const auto kTimeoutHttpMaxStreamDuration =
grpc_core::Duration::Milliseconds(3500);
const auto kTimeoutApplication = grpc_core::Duration::Milliseconds(4500);
const char* kNewCluster1Name = "new_cluster_1";
const char* kNewEdsService1Name = "new_eds_service_name_1";
const char* kNewCluster2Name = "new_cluster_2";
@ -1928,11 +1928,10 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
&http_connection_manager);
// Set up HTTP max_stream_duration of 3.5 seconds
auto* duration =
SetProtoDuration(
kTimeoutHttpMaxStreamDuration,
http_connection_manager.mutable_common_http_protocol_options()
->mutable_max_stream_duration();
duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
duration->set_nanos(kTimeoutNano);
->mutable_max_stream_duration());
listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
http_connection_manager);
// Construct route config.
@ -1944,20 +1943,17 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
route1->mutable_route()->set_cluster(kNewCluster1Name);
auto* max_stream_duration =
route1->mutable_route()->mutable_max_stream_duration();
duration = max_stream_duration->mutable_max_stream_duration();
duration->set_seconds(kTimeoutMaxStreamDurationSecond);
duration->set_nanos(kTimeoutNano);
duration = max_stream_duration->mutable_grpc_timeout_header_max();
duration->set_seconds(kTimeoutGrpcTimeoutHeaderMaxSecond);
duration->set_nanos(kTimeoutNano);
SetProtoDuration(kTimeoutMaxStreamDuration,
max_stream_duration->mutable_max_stream_duration());
SetProtoDuration(kTimeoutGrpcHeaderMax,
max_stream_duration->mutable_grpc_timeout_header_max());
// route 2: Set max_stream_duration of 2.5 seconds
auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
route2->mutable_route()->set_cluster(kNewCluster2Name);
max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
duration = max_stream_duration->mutable_max_stream_duration();
duration->set_seconds(kTimeoutMaxStreamDurationSecond);
duration->set_nanos(kTimeoutNano);
SetProtoDuration(kTimeoutMaxStreamDuration,
max_stream_duration->mutable_max_stream_duration());
// route 3: No timeout values in route configuration
auto* route3 = new_route_config.mutable_virtual_hosts(0)->add_routes();
route3->mutable_match()->set_path("/grpc.testing.EchoTestService/Echo");
@ -1968,55 +1964,45 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
// Test grpc_timeout_header_max of 1.5 seconds applied
grpc_core::Timestamp t0 = NowFromCycleCounter();
grpc_core::Timestamp t1 =
t0 + grpc_core::Duration::Seconds(kTimeoutGrpcTimeoutHeaderMaxSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
t0 + (kTimeoutGrpcHeaderMax * grpc_test_slowdown_factor());
grpc_core::Timestamp t2 =
t0 + grpc_core::Duration::Seconds(kTimeoutMaxStreamDurationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
t0 + (kTimeoutMaxStreamDuration * grpc_test_slowdown_factor());
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond)
.millis()));
.set_timeout(kTimeoutApplication));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
// Test max_stream_duration of 2.5 seconds applied
t0 = NowFromCycleCounter();
t1 = t0 + grpc_core::Duration::Seconds(kTimeoutMaxStreamDurationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
t2 = t0 + grpc_core::Duration::Seconds(kTimeoutHttpMaxStreamDurationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
t1 = t0 + (kTimeoutMaxStreamDuration * grpc_test_slowdown_factor());
t2 = t0 + (kTimeoutHttpMaxStreamDuration * grpc_test_slowdown_factor());
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
"Deadline Exceeded",
RpcOptions()
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond)
.millis()));
.set_timeout(kTimeoutApplication));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
// Test http_stream_duration of 3.5 seconds applied
t0 = NowFromCycleCounter();
t1 = t0 + grpc_core::Duration::Seconds(kTimeoutHttpMaxStreamDurationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
t2 = t0 + grpc_core::Duration::Seconds(kTimeoutApplicationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
t1 = t0 + (kTimeoutHttpMaxStreamDuration * grpc_test_slowdown_factor());
t2 = t0 + (kTimeoutApplication * grpc_test_slowdown_factor());
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
RpcOptions().set_wait_for_ready(true).set_timeout(kTimeoutApplication));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
}
TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
const int64_t kTimeoutNano = 500000000;
const int64_t kTimeoutMaxStreamDurationSecond = 2;
const int64_t kTimeoutHttpMaxStreamDurationSecond = 3;
const int64_t kTimeoutApplicationSecond = 4;
TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit) {
const auto kTimeoutMaxStreamDuration =
grpc_core::Duration::Milliseconds(2500);
const auto kTimeoutHttpMaxStreamDuration =
grpc_core::Duration::Milliseconds(3500);
const auto kTimeoutApplication = grpc_core::Duration::Milliseconds(4500);
const char* kNewCluster1Name = "new_cluster_1";
const char* kNewEdsService1Name = "new_eds_service_name_1";
const char* kNewCluster2Name = "new_cluster_2";
@ -2047,11 +2033,10 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
&http_connection_manager);
// Set up HTTP max_stream_duration of 3.5 seconds
auto* duration =
SetProtoDuration(
kTimeoutHttpMaxStreamDuration,
http_connection_manager.mutable_common_http_protocol_options()
->mutable_max_stream_duration();
duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
duration->set_nanos(kTimeoutNano);
->mutable_max_stream_duration());
listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
http_connection_manager);
// Construct route config.
@ -2063,20 +2048,17 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
route1->mutable_route()->set_cluster(kNewCluster1Name);
auto* max_stream_duration =
route1->mutable_route()->mutable_max_stream_duration();
duration = max_stream_duration->mutable_max_stream_duration();
duration->set_seconds(kTimeoutMaxStreamDurationSecond);
duration->set_nanos(kTimeoutNano);
duration = max_stream_duration->mutable_grpc_timeout_header_max();
duration->set_seconds(0);
duration->set_nanos(0);
SetProtoDuration(kTimeoutMaxStreamDuration,
max_stream_duration->mutable_max_stream_duration());
SetProtoDuration(grpc_core::Duration::Zero(),
max_stream_duration->mutable_grpc_timeout_header_max());
// route 2: Set max_stream_duration to 0
auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
route2->mutable_route()->set_cluster(kNewCluster2Name);
max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
duration = max_stream_duration->mutable_max_stream_duration();
duration->set_seconds(0);
duration->set_nanos(0);
SetProtoDuration(grpc_core::Duration::Zero(),
max_stream_duration->mutable_max_stream_duration());
// Set listener and route config.
SetListenerAndRouteConfiguration(balancer_.get(), std::move(listener),
new_route_config);
@ -2088,12 +2070,13 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
.set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000));
auto ellapsed_nano_seconds =
.set_timeout(kTimeoutApplication));
auto elapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0);
EXPECT_GT(ellapsed_nano_seconds.count(),
kTimeoutApplicationSecond * 1000000000);
EXPECT_GT(
elapsed_nano_seconds.count(),
(kTimeoutApplication * grpc_test_slowdown_factor()).millis() * 1000);
// Test application timeout is applied for route 2
t0 = system_clock::now();
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
@ -2102,15 +2085,16 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
.set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000));
ellapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
.set_timeout(kTimeoutApplication));
elapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
system_clock::now() - t0);
EXPECT_GT(ellapsed_nano_seconds.count(),
kTimeoutApplicationSecond * 1000000000);
EXPECT_GT(
elapsed_nano_seconds.count(),
(kTimeoutApplication * grpc_test_slowdown_factor()).millis() * 1000);
}
TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) {
const int64_t kTimeoutApplicationSecond = 4;
TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit) {
const auto kTimeoutApplication = grpc_core::Duration::Milliseconds(4500);
// Populate new EDS resources.
EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
@ -2133,32 +2117,32 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) {
auto t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
auto ellapsed_nano_seconds =
RpcOptions().set_wait_for_ready(true).set_timeout(kTimeoutApplication));
auto elapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0);
EXPECT_GT(ellapsed_nano_seconds.count(),
kTimeoutApplicationSecond * 1000000000);
EXPECT_GT(
elapsed_nano_seconds.count(),
(kTimeoutApplication * grpc_test_slowdown_factor()).millis() * 1000);
}
// Test to ensure application-specified deadline won't be affected when
// the xDS config does not specify a timeout.
TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) {
const int64_t kTimeoutApplicationSecond = 4;
const auto kTimeoutApplication = grpc_core::Duration::Milliseconds(4500);
// Populate new EDS resources.
EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
auto t0 = system_clock::now();
CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
auto ellapsed_nano_seconds =
RpcOptions().set_wait_for_ready(true).set_timeout(kTimeoutApplication));
auto elapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0);
EXPECT_GT(ellapsed_nano_seconds.count(),
kTimeoutApplicationSecond * 1000000000);
EXPECT_GT(
elapsed_nano_seconds.count(),
(kTimeoutApplication * grpc_test_slowdown_factor()).millis() * 1000);
}
TEST_P(LdsRdsTest, XdsRetryPolicyNumRetries) {
@ -2252,11 +2236,10 @@ TEST_P(LdsRdsTest, XdsRetryPolicyLongBackOff) {
"5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
"unavailable");
retry_policy->mutable_num_retries()->set_value(kNumRetries);
auto base_interval =
retry_policy->mutable_retry_back_off()->mutable_base_interval();
// Set backoff to 1 second, 1/2 of rpc timeout of 2 second.
base_interval->set_seconds(1 * grpc_test_slowdown_factor());
base_interval->set_nanos(0);
SetProtoDuration(
grpc_core::Duration::Seconds(1),
retry_policy->mutable_retry_back_off()->mutable_base_interval());
SetRouteConfiguration(balancer_.get(), new_route_config);
// No need to set max interval and just let it be the default of 10x of base.
// We expect 1 retry before the RPC times out with DEADLINE_EXCEEDED.
@ -2285,19 +2268,17 @@ TEST_P(LdsRdsTest, XdsRetryPolicyMaxBackOff) {
"5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
"unavailable");
retry_policy->mutable_num_retries()->set_value(kNumRetries);
auto base_interval =
retry_policy->mutable_retry_back_off()->mutable_base_interval();
// Set backoff to 1 second.
base_interval->set_seconds(1 * grpc_test_slowdown_factor());
base_interval->set_nanos(0);
auto max_interval =
retry_policy->mutable_retry_back_off()->mutable_max_interval();
SetProtoDuration(
grpc_core::Duration::Seconds(1),
retry_policy->mutable_retry_back_off()->mutable_base_interval());
// Set max interval to be the same as base, so 2 retries will take 2 seconds
// and both retries will take place before the 2.5 seconds rpc timeout.
// Tested to ensure if max is not set, this test will be the same as
// XdsRetryPolicyLongBackOff and we will only see 1 retry in that case.
max_interval->set_seconds(1 * grpc_test_slowdown_factor());
max_interval->set_nanos(0);
SetProtoDuration(
grpc_core::Duration::Seconds(1),
retry_policy->mutable_retry_back_off()->mutable_max_interval());
SetRouteConfiguration(balancer_.get(), new_route_config);
// Send an initial RPC to make sure we get connected (we don't want
// the channel startup time to affect the retry timing).
@ -2400,10 +2381,9 @@ TEST_P(LdsRdsTest, XdsRetryPolicyRetryBackOffMissingBaseInterval) {
retry_policy->set_retry_on("deadline-exceeded");
retry_policy->mutable_num_retries()->set_value(1);
// RetryBackoff is there but base interval is missing.
auto max_interval =
retry_policy->mutable_retry_back_off()->mutable_max_interval();
max_interval->set_seconds(0);
max_interval->set_nanos(250000000);
SetProtoDuration(
grpc_core::Duration::Milliseconds(250),
retry_policy->mutable_retry_back_off()->mutable_max_interval());
SetRouteConfiguration(balancer_.get(), new_route_config);
const auto response_state = WaitForRdsNack(DEBUG_LOCATION);
ASSERT_TRUE(response_state.has_value()) << "timed out waiting for NACK";

3
test/cpp/end2end/xds/xds_server.h
Unescape View File

@ -853,7 +853,8 @@ class LrsServiceImpl : public std::enable_shared_from_this<LrsServiceImpl> {
}
}
response.mutable_load_reporting_interval()->set_seconds(
parent_->client_load_reporting_interval_seconds_);
parent_->client_load_reporting_interval_seconds_ *
grpc_test_slowdown_factor());
stream->Write(response);
CountedService<typename RpcApi::Service>::IncreaseResponseCount();
// Wait for report.

Write Preview
Loading…
Cancel
Save