end2end tests: apply test slowdown factor in various places where it was missed (#30749)

* e2e tests: add test scaling factor to durations in channel args

* apply test scaling factor when encoding durations in xDS protos

* apply scaling factor in fixed timeout in WaitForNack()

* fix overflow

* clang-format

* adjust timeouts in fault injection tests

* add missing slowdown factor

* clang-format

* add tests for duration multiplication
pull/30756/head^2
Mark D. Roth 2 years ago committed by GitHub
parent 8b941bb648
commit 121a08f6a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 6
      src/core/lib/gprpp/time.h
  2. 7
      test/core/gprpp/time_test.cc
  3. 32
      test/cpp/end2end/client_lb_end2end_test.cc
  4. 14
      test/cpp/end2end/xds/xds_end2end_test_lib.cc
  5. 11
      test/cpp/end2end/xds/xds_end2end_test_lib.h
  6. 99
      test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc
  7. 397
      test/cpp/end2end/xds/xds_outlier_detection_end2end_test.cc
  8. 3
      test/cpp/end2end/xds/xds_ring_hash_end2end_test.cc
  9. 192
      test/cpp/end2end/xds/xds_routing_end2end_test.cc
  10. 3
      test/cpp/end2end/xds/xds_server.h

@ -200,6 +200,7 @@ class Duration {
} }
return *this; return *this;
} }
Duration& operator*=(double multiplier);
Duration& operator+=(Duration other) { Duration& operator+=(Duration other) {
millis_ += other.millis_; millis_ += other.millis_;
return *this; return *this;
@ -287,6 +288,11 @@ inline Duration Duration::FromSecondsAsDouble(double seconds) {
return Milliseconds(static_cast<int64_t>(millis)); return Milliseconds(static_cast<int64_t>(millis));
} }
inline Duration& Duration::operator*=(double multiplier) {
*this = *this * multiplier;
return *this;
}
inline Timestamp& Timestamp::operator+=(Duration duration) { inline Timestamp& Timestamp::operator+=(Duration duration) {
return *this = (*this + duration); return *this = (*this + duration);
} }

@ -75,6 +75,13 @@ TEST(DurationTest, Infinities) {
Duration::NegativeInfinity()); Duration::NegativeInfinity());
} }
TEST(DurationTest, Multiplication) {
Duration d = Duration::Seconds(5);
EXPECT_EQ(d * 2, Duration::Seconds(10));
d *= 3;
EXPECT_EQ(d, Duration::Seconds(15));
}
TEST(DurationTest, FromTimespan) { TEST(DurationTest, FromTimespan) {
EXPECT_EQ(Duration::FromTimespec(gpr_time_from_millis(1234, GPR_TIMESPAN)), EXPECT_EQ(Duration::FromTimespec(gpr_time_from_millis(1234, GPR_TIMESPAN)),
Duration::Milliseconds(1234)); Duration::Milliseconds(1234));

@ -581,7 +581,8 @@ TEST_F(ClientLbEnd2endTest, ChannelIdleness) {
StartServers(kNumServers); StartServers(kNumServers);
// Set max idle time and build the channel. // Set max idle time and build the channel.
ChannelArguments args; ChannelArguments args;
args.SetInt(GRPC_ARG_CLIENT_IDLE_TIMEOUT_MS, 1000); args.SetInt(GRPC_ARG_CLIENT_IDLE_TIMEOUT_MS,
1000 * grpc_test_slowdown_factor());
auto response_generator = BuildResolverResponseGenerator(); auto response_generator = BuildResolverResponseGenerator();
auto channel = BuildChannel("", response_generator, args); auto channel = BuildChannel("", response_generator, args);
auto stub = BuildStub(channel); auto stub = BuildStub(channel);
@ -732,7 +733,8 @@ TEST_F(PickFirstTest, ProcessPending) {
TEST_F(PickFirstTest, SelectsReadyAtStartup) { TEST_F(PickFirstTest, SelectsReadyAtStartup) {
ChannelArguments args; ChannelArguments args;
constexpr int kInitialBackOffMs = 5000; constexpr int kInitialBackOffMs = 5000;
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs); args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
kInitialBackOffMs * grpc_test_slowdown_factor());
// Create 2 servers, but start only the second one. // Create 2 servers, but start only the second one.
std::vector<int> ports = {grpc_pick_unused_port_or_die(), std::vector<int> ports = {grpc_pick_unused_port_or_die(),
grpc_pick_unused_port_or_die()}; grpc_pick_unused_port_or_die()};
@ -758,7 +760,8 @@ TEST_F(PickFirstTest, SelectsReadyAtStartup) {
TEST_F(PickFirstTest, BackOffInitialReconnect) { TEST_F(PickFirstTest, BackOffInitialReconnect) {
ChannelArguments args; ChannelArguments args;
constexpr int kInitialBackOffMs = 100; constexpr int kInitialBackOffMs = 100;
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs); args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
kInitialBackOffMs * grpc_test_slowdown_factor());
const std::vector<int> ports = {grpc_pick_unused_port_or_die()}; const std::vector<int> ports = {grpc_pick_unused_port_or_die()};
const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC); const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC);
auto response_generator = BuildResolverResponseGenerator(); auto response_generator = BuildResolverResponseGenerator();
@ -779,7 +782,8 @@ TEST_F(PickFirstTest, BackOffInitialReconnect) {
gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited.millis()); gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited.millis());
// We should have waited at least kInitialBackOffMs. We substract one to // We should have waited at least kInitialBackOffMs. We substract one to
// account for test and precision accuracy drift. // account for test and precision accuracy drift.
EXPECT_GE(waited.millis(), kInitialBackOffMs - 1); EXPECT_GE(waited.millis(),
(kInitialBackOffMs * grpc_test_slowdown_factor()) - 1);
// But not much more. // But not much more.
EXPECT_GT( EXPECT_GT(
gpr_time_cmp( gpr_time_cmp(
@ -790,7 +794,8 @@ TEST_F(PickFirstTest, BackOffInitialReconnect) {
TEST_F(PickFirstTest, BackOffMinReconnect) { TEST_F(PickFirstTest, BackOffMinReconnect) {
ChannelArguments args; ChannelArguments args;
constexpr int kMinReconnectBackOffMs = 1000; constexpr int kMinReconnectBackOffMs = 1000;
args.SetInt(GRPC_ARG_MIN_RECONNECT_BACKOFF_MS, kMinReconnectBackOffMs); args.SetInt(GRPC_ARG_MIN_RECONNECT_BACKOFF_MS,
kMinReconnectBackOffMs * grpc_test_slowdown_factor());
const std::vector<int> ports = {grpc_pick_unused_port_or_die()}; const std::vector<int> ports = {grpc_pick_unused_port_or_die()};
auto response_generator = BuildResolverResponseGenerator(); auto response_generator = BuildResolverResponseGenerator();
auto channel = BuildChannel("pick_first", response_generator, args); auto channel = BuildChannel("pick_first", response_generator, args);
@ -799,8 +804,8 @@ TEST_F(PickFirstTest, BackOffMinReconnect) {
// Make connection delay a 10% longer than it's willing to in order to make // Make connection delay a 10% longer than it's willing to in order to make
// sure we are hitting the codepath that waits for the min reconnect backoff. // sure we are hitting the codepath that waits for the min reconnect backoff.
ConnectionAttemptInjector injector; ConnectionAttemptInjector injector;
injector.SetDelay( injector.SetDelay(grpc_core::Duration::Milliseconds(
grpc_core::Duration::Milliseconds(kMinReconnectBackOffMs * 1.10)); kMinReconnectBackOffMs * grpc_test_slowdown_factor() * 1.10));
const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC); const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC);
channel->WaitForConnected( channel->WaitForConnected(
grpc_timeout_milliseconds_to_deadline(kMinReconnectBackOffMs * 2)); grpc_timeout_milliseconds_to_deadline(kMinReconnectBackOffMs * 2));
@ -810,13 +815,15 @@ TEST_F(PickFirstTest, BackOffMinReconnect) {
gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited.millis()); gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited.millis());
// We should have waited at least kMinReconnectBackOffMs. We substract one to // We should have waited at least kMinReconnectBackOffMs. We substract one to
// account for test and precision accuracy drift. // account for test and precision accuracy drift.
EXPECT_GE(waited.millis(), kMinReconnectBackOffMs - 1); EXPECT_GE(waited.millis(),
(kMinReconnectBackOffMs * grpc_test_slowdown_factor()) - 1);
} }
TEST_F(PickFirstTest, ResetConnectionBackoff) { TEST_F(PickFirstTest, ResetConnectionBackoff) {
ChannelArguments args; ChannelArguments args;
constexpr int kInitialBackOffMs = 1000; constexpr int kInitialBackOffMs = 1000;
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs); args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
kInitialBackOffMs * grpc_test_slowdown_factor());
const std::vector<int> ports = {grpc_pick_unused_port_or_die()}; const std::vector<int> ports = {grpc_pick_unused_port_or_die()};
auto response_generator = BuildResolverResponseGenerator(); auto response_generator = BuildResolverResponseGenerator();
auto channel = BuildChannel("pick_first", response_generator, args); auto channel = BuildChannel("pick_first", response_generator, args);
@ -844,7 +851,7 @@ TEST_F(PickFirstTest, ResetConnectionBackoff) {
grpc_core::Duration::FromTimespec(gpr_time_sub(t1, t0)); grpc_core::Duration::FromTimespec(gpr_time_sub(t1, t0));
gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited.millis()); gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited.millis());
// We should have waited less than kInitialBackOffMs. // We should have waited less than kInitialBackOffMs.
EXPECT_LT(waited.millis(), kInitialBackOffMs); EXPECT_LT(waited.millis(), kInitialBackOffMs * grpc_test_slowdown_factor());
} }
TEST_F(ClientLbEnd2endTest, TEST_F(ClientLbEnd2endTest,
@ -854,8 +861,9 @@ TEST_F(ClientLbEnd2endTest,
// Create client. // Create client.
const int port = grpc_pick_unused_port_or_die(); const int port = grpc_pick_unused_port_or_die();
ChannelArguments args; ChannelArguments args;
const int kInitialBackOffMs = 5000 * grpc_test_slowdown_factor(); const int kInitialBackOffMs = 5000;
args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs); args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
kInitialBackOffMs * grpc_test_slowdown_factor());
auto response_generator = BuildResolverResponseGenerator(); auto response_generator = BuildResolverResponseGenerator();
auto channel = BuildChannel("pick_first", response_generator, args); auto channel = BuildChannel("pick_first", response_generator, args);
auto stub = BuildStub(channel); auto stub = BuildStub(channel);

@ -791,7 +791,8 @@ std::shared_ptr<Channel> XdsEnd2endTest::CreateChannel(
// TODO(roth): Remove this once we enable retries by default internally. // TODO(roth): Remove this once we enable retries by default internally.
args->SetInt(GRPC_ARG_ENABLE_RETRIES, 1); args->SetInt(GRPC_ARG_ENABLE_RETRIES, 1);
if (failover_timeout_ms > 0) { if (failover_timeout_ms > 0) {
args->SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, failover_timeout_ms); args->SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS,
failover_timeout_ms * grpc_test_slowdown_factor());
} }
if (GetParam().bootstrap_source() == XdsTestType::kBootstrapFromChannelArg) { if (GetParam().bootstrap_source() == XdsTestType::kBootstrapFromChannelArg) {
// We're getting the bootstrap from a channel arg, so we do the // We're getting the bootstrap from a channel arg, so we do the
@ -1004,7 +1005,8 @@ absl::optional<AdsServiceImpl::ResponseState> XdsEnd2endTest::WaitForNack(
std::function<absl::optional<AdsServiceImpl::ResponseState>()> get_state, std::function<absl::optional<AdsServiceImpl::ResponseState>()> get_state,
const RpcOptions& rpc_options, StatusCode expected_status) { const RpcOptions& rpc_options, StatusCode expected_status) {
absl::optional<AdsServiceImpl::ResponseState> response_state; absl::optional<AdsServiceImpl::ResponseState> response_state;
auto deadline = absl::Now() + absl::Seconds(30); auto deadline =
absl::Now() + (absl::Seconds(30) * grpc_test_slowdown_factor());
auto continue_predicate = [&]() { auto continue_predicate = [&]() {
if (absl::Now() >= deadline) { if (absl::Now() >= deadline) {
return false; return false;
@ -1023,6 +1025,14 @@ absl::optional<AdsServiceImpl::ResponseState> XdsEnd2endTest::WaitForNack(
return response_state; return response_state;
} }
void XdsEnd2endTest::SetProtoDuration(
grpc_core::Duration duration, google::protobuf::Duration* duration_proto) {
duration *= grpc_test_slowdown_factor();
gpr_timespec ts = duration.as_timespec();
duration_proto->set_seconds(ts.tv_sec);
duration_proto->set_nanos(ts.tv_nsec);
}
std::string XdsEnd2endTest::ReadFile(const char* file_path) { std::string XdsEnd2endTest::ReadFile(const char* file_path) {
grpc_slice slice; grpc_slice slice;
GPR_ASSERT( GPR_ASSERT(

@ -720,6 +720,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
// Dev note: If a timeout or Deadline Exceeded error is occurring in an XDS // Dev note: If a timeout or Deadline Exceeded error is occurring in an XDS
// end2end test, consider changing that test's timeout instead of this // end2end test, consider changing that test's timeout instead of this
// global default. // global default.
// Will be multiplied by grpc_test_slowdown_factor().
int timeout_ms = 1000; int timeout_ms = 1000;
bool wait_for_ready = false; bool wait_for_ready = false;
std::vector<std::pair<std::string, std::string>> metadata; std::vector<std::pair<std::string, std::string>> metadata;
@ -747,6 +748,11 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
return *this; return *this;
} }
RpcOptions& set_timeout(grpc_core::Duration timeout) {
timeout_ms = timeout.millis();
return *this;
}
RpcOptions& set_wait_for_ready(bool rpc_wait_for_ready) { RpcOptions& set_wait_for_ready(bool rpc_wait_for_ready) {
wait_for_ready = rpc_wait_for_ready; wait_for_ready = rpc_wait_for_ready;
return *this; return *this;
@ -884,6 +890,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
// If true, resets the backend counters before returning. // If true, resets the backend counters before returning.
bool reset_counters = true; bool reset_counters = true;
// How long to wait for the backend(s) to see requests. // How long to wait for the backend(s) to see requests.
// Will be multiplied by grpc_test_slowdown_factor().
int timeout_ms = 5000; int timeout_ms = 5000;
WaitForBackendOptions() {} WaitForBackendOptions() {}
@ -1012,6 +1019,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam<XdsTestType> {
gpr_now(GPR_CLOCK_MONOTONIC)); gpr_now(GPR_CLOCK_MONOTONIC));
} }
// Sets duration_proto to duration times grpc_test_slowdown_factor().
static void SetProtoDuration(grpc_core::Duration duration,
google::protobuf::Duration* duration_proto);
// Returns the number of RPCs needed to pass error_tolerance at 99.99994% // Returns the number of RPCs needed to pass error_tolerance at 99.99994%
// chance. Rolling dices in drop/fault-injection generates a binomial // chance. Rolling dices in drop/fault-injection generates a binomial
// distribution (if our code is not horribly wrong). Let's make "n" the number // distribution (if our code is not horribly wrong). Let's make "n" the number

@ -204,8 +204,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) {
TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) { TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
CreateAndStartBackends(1); CreateAndStartBackends(1);
const uint32_t kRpcTimeoutMilliseconds = 10000; const auto kRpcTimeout = grpc_core::Duration::Seconds(10);
const uint32_t kFixedDelaySeconds = 100; const auto kFixedDelay = grpc_core::Duration::Seconds(20);
const uint32_t kDelayPercentagePerHundred = 50; const uint32_t kDelayPercentagePerHundred = 50;
const double kDelayRate = kDelayPercentagePerHundred / 100.0; const double kDelayRate = kDelayPercentagePerHundred / 100.0;
const double kErrorTolerance = 0.1; const double kErrorTolerance = 0.1;
@ -225,14 +225,13 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(kDelayPercentagePerHundred); delay_percentage->set_numerator(kDelayPercentagePerHundred);
delay_percentage->set_denominator(FractionalPercent::HUNDRED); delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); SetProtoDuration(kFixedDelay,
fixed_delay->set_seconds(kFixedDelaySeconds); http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup // Config fault injection via different setup
SetFilterConfig(http_fault); SetFilterConfig(http_fault);
// Send kNumRpcs RPCs and count the delays. // Send kNumRpcs RPCs and count the delays.
RpcOptions rpc_options = RpcOptions() RpcOptions rpc_options =
.set_timeout_ms(kRpcTimeoutMilliseconds) RpcOptions().set_timeout(kRpcTimeout).set_skip_cancelled_check(true);
.set_skip_cancelled_check(true);
std::vector<ConcurrentRpc> rpcs = std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options); SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
size_t num_delayed = 0; size_t num_delayed = 0;
@ -249,8 +248,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) { TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
CreateAndStartBackends(1); CreateAndStartBackends(1);
const uint32_t kFixedDelayMilliseconds = 100000; const auto kRpcTimeout = grpc_core::Duration::Seconds(10);
const uint32_t kRpcTimeoutMilliseconds = 10000; const auto kFixedDelay = grpc_core::Duration::Seconds(20);
const uint32_t kDelayPercentageCap = 100; const uint32_t kDelayPercentageCap = 100;
const uint32_t kDelayPercentage = 50; const uint32_t kDelayPercentage = 50;
const double kDelayRate = kDelayPercentage / 100.0; const double kDelayRate = kDelayPercentage / 100.0;
@ -275,13 +274,14 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
SetFilterConfig(http_fault); SetFilterConfig(http_fault);
// Send kNumRpcs RPCs and count the delays. // Send kNumRpcs RPCs and count the delays.
std::vector<std::pair<std::string, std::string>> metadata = { std::vector<std::pair<std::string, std::string>> metadata = {
{"x-envoy-fault-delay-request", std::to_string(kFixedDelayMilliseconds)}, {"x-envoy-fault-delay-request",
std::to_string(kFixedDelay.millis() * grpc_test_slowdown_factor())},
{"x-envoy-fault-delay-request-percentage", {"x-envoy-fault-delay-request-percentage",
std::to_string(kDelayPercentage)}, std::to_string(kDelayPercentage)},
}; };
RpcOptions rpc_options = RpcOptions() RpcOptions rpc_options = RpcOptions()
.set_metadata(metadata) .set_metadata(metadata)
.set_timeout_ms(kRpcTimeoutMilliseconds) .set_timeout(kRpcTimeout)
.set_skip_cancelled_check(true); .set_skip_cancelled_check(true);
std::vector<ConcurrentRpc> rpcs = std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options); SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
@ -299,8 +299,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) { TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) {
CreateAndStartBackends(1); CreateAndStartBackends(1);
const uint32_t kFixedDelaySeconds = 1; const auto kRpcTimeout = grpc_core::Duration::Seconds(30);
const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach const auto kFixedDelay = grpc_core::Duration::Seconds(1);
// Create an EDS resource // Create an EDS resource
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
@ -314,14 +314,14 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(100); // Always inject DELAY! delay_percentage->set_numerator(100); // Always inject DELAY!
delay_percentage->set_denominator(FractionalPercent::HUNDRED); delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); SetProtoDuration(kFixedDelay,
fixed_delay->set_seconds(kFixedDelaySeconds); http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup // Config fault injection via different setup
SetFilterConfig(http_fault); SetFilterConfig(http_fault);
// Send a stream RPC and check its status code // Send a stream RPC and check its status code
ClientContext context; ClientContext context;
context.set_deadline( context.set_deadline(
grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds)); grpc_timeout_milliseconds_to_deadline(kRpcTimeout.millis()));
auto stream = stub_->BidiStream(&context); auto stream = stub_->BidiStream(&context);
stream->WritesDone(); stream->WritesDone();
auto status = stream->Finish(); auto status = stream->Finish();
@ -332,12 +332,11 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) {
TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) { TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
CreateAndStartBackends(1); CreateAndStartBackends(1);
const auto kConnectTimeout = grpc_core::Duration::Seconds(10);
const auto kRpcTimeout = grpc_core::Duration::Seconds(30);
const auto kFixedDelay = grpc_core::Duration::Seconds(1);
const uint32_t kAbortPercentagePerHundred = 50; const uint32_t kAbortPercentagePerHundred = 50;
const double kAbortRate = kAbortPercentagePerHundred / 100.0; const double kAbortRate = kAbortPercentagePerHundred / 100.0;
const uint32_t kFixedDelaySeconds = 1;
const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach
const uint32_t kConnectionTimeoutMilliseconds =
10 * 1000; // 10s should not reach
const double kErrorTolerance = 0.1; const double kErrorTolerance = 0.1;
const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
const size_t kMaxConcurrentRequests = kNumRpcs; const size_t kMaxConcurrentRequests = kNumRpcs;
@ -360,23 +359,22 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(1000000); // Always inject DELAY! delay_percentage->set_numerator(1000000); // Always inject DELAY!
delay_percentage->set_denominator(FractionalPercent::MILLION); delay_percentage->set_denominator(FractionalPercent::MILLION);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); SetProtoDuration(kFixedDelay,
fixed_delay->set_seconds(kFixedDelaySeconds); http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup // Config fault injection via different setup
SetFilterConfig(http_fault); SetFilterConfig(http_fault);
// Allow the channel to connect to one backends, so the herd of queued RPCs // Allow the channel to connect to one backends, so the herd of queued RPCs
// won't be executed on the same ExecCtx object and using the cached Now() // won't be executed on the same ExecCtx object and using the cached Now()
// value, which causes millisecond level delay error. // value, which causes millisecond level delay error.
channel_->WaitForConnected( channel_->WaitForConnected(
grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)); grpc_timeout_milliseconds_to_deadline(kConnectTimeout.millis()));
// Send kNumRpcs RPCs and count the aborts. // Send kNumRpcs RPCs and count the aborts.
int num_aborted = 0; int num_aborted = 0;
RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds); RpcOptions rpc_options = RpcOptions().set_timeout(kRpcTimeout);
std::vector<ConcurrentRpc> rpcs = std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options); SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
for (auto& rpc : rpcs) { for (auto& rpc : rpcs) {
EXPECT_GE(rpc.elapsed_time, EXPECT_GE(rpc.elapsed_time, kFixedDelay * grpc_test_slowdown_factor());
grpc_core::Duration::Seconds(kFixedDelaySeconds));
if (rpc.status.error_code() == StatusCode::OK) continue; if (rpc.status.error_code() == StatusCode::OK) continue;
EXPECT_EQ("Fault injected", rpc.status.error_message()); EXPECT_EQ("Fault injected", rpc.status.error_message());
++num_aborted; ++num_aborted;
@ -393,12 +391,11 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
TEST_P(FaultInjectionTest, TEST_P(FaultInjectionTest,
XdsFaultInjectionAlwaysDelayPercentageAbortSwitchDenominator) { XdsFaultInjectionAlwaysDelayPercentageAbortSwitchDenominator) {
CreateAndStartBackends(1); CreateAndStartBackends(1);
const auto kConnectTimeout = grpc_core::Duration::Seconds(10);
const auto kRpcTimeout = grpc_core::Duration::Seconds(30);
const auto kFixedDelay = grpc_core::Duration::Seconds(1);
const uint32_t kAbortPercentagePerMillion = 500000; const uint32_t kAbortPercentagePerMillion = 500000;
const double kAbortRate = kAbortPercentagePerMillion / 1000000.0; const double kAbortRate = kAbortPercentagePerMillion / 1000000.0;
const uint32_t kFixedDelaySeconds = 1; // 1s
const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach
const uint32_t kConnectionTimeoutMilliseconds =
10 * 1000; // 10s should not reach
const double kErrorTolerance = 0.1; const double kErrorTolerance = 0.1;
const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
const size_t kMaxConcurrentRequests = kNumRpcs; const size_t kMaxConcurrentRequests = kNumRpcs;
@ -421,23 +418,22 @@ TEST_P(FaultInjectionTest,
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(100); // Always inject DELAY! delay_percentage->set_numerator(100); // Always inject DELAY!
delay_percentage->set_denominator(FractionalPercent::HUNDRED); delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); SetProtoDuration(kFixedDelay,
fixed_delay->set_seconds(kFixedDelaySeconds); http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup // Config fault injection via different setup
SetFilterConfig(http_fault); SetFilterConfig(http_fault);
// Allow the channel to connect to one backends, so the herd of queued RPCs // Allow the channel to connect to one backends, so the herd of queued RPCs
// won't be executed on the same ExecCtx object and using the cached Now() // won't be executed on the same ExecCtx object and using the cached Now()
// value, which causes millisecond level delay error. // value, which causes millisecond level delay error.
channel_->WaitForConnected( channel_->WaitForConnected(
grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)); grpc_timeout_milliseconds_to_deadline(kConnectTimeout.millis()));
// Send kNumRpcs RPCs and count the aborts. // Send kNumRpcs RPCs and count the aborts.
int num_aborted = 0; int num_aborted = 0;
RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds); RpcOptions rpc_options = RpcOptions().set_timeout(kRpcTimeout);
std::vector<ConcurrentRpc> rpcs = std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options); SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
for (auto& rpc : rpcs) { for (auto& rpc : rpcs) {
EXPECT_GE(rpc.elapsed_time, EXPECT_GE(rpc.elapsed_time, kFixedDelay * grpc_test_slowdown_factor());
grpc_core::Duration::Seconds(kFixedDelaySeconds));
if (rpc.status.error_code() == StatusCode::OK) continue; if (rpc.status.error_code() == StatusCode::OK) continue;
EXPECT_EQ("Fault injected", rpc.status.error_message()); EXPECT_EQ("Fault injected", rpc.status.error_message());
++num_aborted; ++num_aborted;
@ -450,10 +446,10 @@ TEST_P(FaultInjectionTest,
TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) { TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
CreateAndStartBackends(1); CreateAndStartBackends(1);
const auto kRpcTimeout = grpc_core::Duration::Seconds(4);
const auto kFixedDelay = grpc_core::Duration::Seconds(20);
const uint32_t kMaxFault = 10; const uint32_t kMaxFault = 10;
const uint32_t kNumRpcs = 30; // kNumRpcs should be bigger than kMaxFault const uint32_t kNumRpcs = 30; // kNumRpcs should be bigger than kMaxFault
const uint32_t kRpcTimeoutMs = 4000; // 4 seconds
const uint32_t kLongDelaySeconds = 100; // 100 seconds
const uint32_t kAlwaysDelayPercentage = 100; const uint32_t kAlwaysDelayPercentage = 100;
// Create an EDS resource // Create an EDS resource
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
@ -464,15 +460,15 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
delay_percentage->set_numerator( delay_percentage->set_numerator(
kAlwaysDelayPercentage); // Always inject DELAY! kAlwaysDelayPercentage); // Always inject DELAY!
delay_percentage->set_denominator(FractionalPercent::HUNDRED); delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); SetProtoDuration(kFixedDelay,
fixed_delay->set_seconds(kLongDelaySeconds); http_fault.mutable_delay()->mutable_fixed_delay());
http_fault.mutable_max_active_faults()->set_value(kMaxFault); http_fault.mutable_max_active_faults()->set_value(kMaxFault);
// Config fault injection via different setup // Config fault injection via different setup
SetFilterConfig(http_fault); SetFilterConfig(http_fault);
// Sends a batch of long running RPCs with long timeout to consume all // Sends a batch of long running RPCs with long timeout to consume all
// active faults quota. // active faults quota.
int num_delayed = 0; int num_delayed = 0;
RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMs); RpcOptions rpc_options = RpcOptions().set_timeout(kRpcTimeout);
std::vector<ConcurrentRpc> rpcs = std::vector<ConcurrentRpc> rpcs =
SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options); SendConcurrentRpcs(DEBUG_LOCATION, stub_.get(), kNumRpcs, rpc_options);
for (auto& rpc : rpcs) { for (auto& rpc : rpcs) {
@ -498,9 +494,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) { TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) {
CreateAndStartBackends(1); CreateAndStartBackends(1);
// kRpcTimeoutMilliseconds is 10s should never be reached. const auto kRpcTimeout = grpc_core::Duration::Seconds(20);
const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 10000; const auto kFixedDelay = grpc_core::Duration::Seconds(1);
const uint32_t kFixedDelaySeconds = 1;
const uint32_t kDelayPercentagePerHundred = 100; const uint32_t kDelayPercentagePerHundred = 100;
// Create an EDS resource // Create an EDS resource
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
@ -510,13 +505,13 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(kDelayPercentagePerHundred); delay_percentage->set_numerator(kDelayPercentagePerHundred);
delay_percentage->set_denominator(FractionalPercent::HUNDRED); delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); SetProtoDuration(kFixedDelay,
fixed_delay->set_seconds(kFixedDelaySeconds); http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup // Config fault injection via different setup
SetFilterConfig(http_fault); SetFilterConfig(http_fault);
ClientContext context; ClientContext context;
context.set_deadline( context.set_deadline(
grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds)); grpc_timeout_milliseconds_to_deadline(kRpcTimeout.millis()));
auto stream = stub_->BidiStream(&context); auto stream = stub_->BidiStream(&context);
stream->WritesDone(); stream->WritesDone();
auto status = stream->Finish(); auto status = stream->Finish();
@ -530,8 +525,8 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) {
// for description. // for description.
TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayError) { TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayError) {
CreateAndStartBackends(1); CreateAndStartBackends(1);
const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 500; const auto kRpcTimeout = grpc_core::Duration::Seconds(10);
const uint32_t kFixedDelaySeconds = 100; const auto kFixedDelay = grpc_core::Duration::Seconds(30);
const uint32_t kDelayPercentagePerHundred = 100; const uint32_t kDelayPercentagePerHundred = 100;
// Create an EDS resource // Create an EDS resource
EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}});
@ -541,13 +536,13 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayError) {
auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
delay_percentage->set_numerator(kDelayPercentagePerHundred); delay_percentage->set_numerator(kDelayPercentagePerHundred);
delay_percentage->set_denominator(FractionalPercent::HUNDRED); delay_percentage->set_denominator(FractionalPercent::HUNDRED);
auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); SetProtoDuration(kFixedDelay,
fixed_delay->set_seconds(kFixedDelaySeconds); http_fault.mutable_delay()->mutable_fixed_delay());
// Config fault injection via different setup // Config fault injection via different setup
SetFilterConfig(http_fault); SetFilterConfig(http_fault);
ClientContext context; ClientContext context;
context.set_deadline( context.set_deadline(
grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds)); grpc_timeout_milliseconds_to_deadline(kRpcTimeout.millis()));
auto stream = stub_->BidiStream(&context); auto stream = stub_->BidiStream(&context);
stream->WritesDone(); stream->WritesDone();
auto status = stream->Finish(); auto status = stream->Finish();

@ -63,23 +63,15 @@ TEST_P(OutlierDetectionTest, SuccessRateEjectionAndUnejection) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
auto* base_time = SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection()->mutable_base_ejection_time(); outlier_detection->mutable_interval());
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
base_time->set_seconds(1 * grpc_test_slowdown_factor()); outlier_detection->mutable_base_ejection_time());
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
->mutable_success_rate_stdev_factor() outlier_detection->mutable_enforcing_success_rate()->set_value(100);
->set_value(100); outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_request_volume()->set_value(1);
->mutable_enforcing_success_rate()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -135,20 +127,13 @@ TEST_P(OutlierDetectionTest, SuccessRateMaxPercent) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection() outlier_detection->mutable_interval());
->mutable_success_rate_stdev_factor() outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
->set_value(100); outlier_detection->mutable_enforcing_success_rate()->set_value(100);
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
->mutable_enforcing_success_rate() outlier_detection->mutable_success_rate_request_volume()->set_value(1);
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -239,29 +224,21 @@ TEST_P(OutlierDetectionTest, SuccessRateStdevFactor) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
auto* base_time = SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection()->mutable_base_ejection_time(); outlier_detection->mutable_interval());
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
base_time->set_seconds(1 * grpc_test_slowdown_factor()); outlier_detection->mutable_base_ejection_time());
// We know a stdev factor of 100 will ensure the ejection occurs, so setting // We know a stdev factor of 100 will ensure the ejection occurs, so setting
// it to something higher like 1000 to test that ejection will not occur. // it to something higher like 1000 to test that ejection will not occur.
// Note this parameter is the only difference between this test and // Note this parameter is the only difference between this test and
// SuccessRateEjectionAndUnejection (ejection portion, value set to 100) and // SuccessRateEjectionAndUnejection (ejection portion, value set to 100) and
// this one value changes means the difference between not ejecting in this // this one value changes means the difference between not ejecting in this
// test and ejecting in the other test. // test and ejecting in the other test.
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_stdev_factor()->set_value(1000);
->mutable_success_rate_stdev_factor() outlier_detection->mutable_enforcing_success_rate()->set_value(100);
->set_value(1000); outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_request_volume()->set_value(1);
->mutable_enforcing_success_rate()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -310,28 +287,20 @@ TEST_P(OutlierDetectionTest, SuccessRateEnforcementPercentage) {
CreateAndStartBackends(2); CreateAndStartBackends(2);
auto cluster = default_cluster_; auto cluster = default_cluster_;
cluster.set_lb_policy(Cluster::RING_HASH); cluster.set_lb_policy(Cluster::RING_HASH);
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
auto* base_time = SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection()->mutable_base_ejection_time(); outlier_detection->mutable_interval());
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
base_time->set_seconds(1 * grpc_test_slowdown_factor()); outlier_detection->mutable_base_ejection_time());
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
->mutable_success_rate_stdev_factor()
->set_value(100);
// Setting enforcing_success_rate to 0 to ensure we will never eject. // Setting enforcing_success_rate to 0 to ensure we will never eject.
// Note this parameter is the only difference between this test and // Note this parameter is the only difference between this test and
// SuccessRateEjectionAndUnejection (ejection portion, value set to 100) and // SuccessRateEjectionAndUnejection (ejection portion, value set to 100) and
// this one value changes means the difference between guaranteed not ejecting // this one value changes means the difference between guaranteed not ejecting
// in this test and guaranteed ejecting in the other test. // in this test and guaranteed ejecting in the other test.
cluster.mutable_outlier_detection() outlier_detection->mutable_enforcing_success_rate()->set_value(0);
->mutable_enforcing_success_rate() outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
->set_value(0); outlier_detection->mutable_success_rate_request_volume()->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -382,25 +351,18 @@ TEST_P(OutlierDetectionTest, SuccessRateMinimumHosts) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection() outlier_detection->mutable_interval());
->mutable_success_rate_stdev_factor() outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
->set_value(100); outlier_detection->mutable_enforcing_success_rate()->set_value(100);
cluster.mutable_outlier_detection()
->mutable_enforcing_success_rate()
->set_value(100);
// Set success_rate_minimum_hosts to 3 when we only have 2 backends // Set success_rate_minimum_hosts to 3 when we only have 2 backends
// Note this parameter is the only difference between this test and // Note this parameter is the only difference between this test and
// SuccessRateEjectionAndUnejection (ejection portion, value set to 1) and // SuccessRateEjectionAndUnejection (ejection portion, value set to 1) and
// this one value changes means the difference between not ejecting in this // this one value changes means the difference between not ejecting in this
// test and ejecting in the other test. // test and ejecting in the other test.
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_minimum_hosts()->set_value(3);
->mutable_success_rate_minimum_hosts() outlier_detection->mutable_success_rate_request_volume()->set_value(1);
->set_value(3);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -451,26 +413,19 @@ TEST_P(OutlierDetectionTest, SuccessRateRequestVolume) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection() outlier_detection->mutable_interval());
->mutable_success_rate_stdev_factor() outlier_detection->mutable_success_rate_stdev_factor()->set_value(100);
->set_value(100); outlier_detection->mutable_enforcing_success_rate()->set_value(100);
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
->mutable_enforcing_success_rate()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_success_rate_minimum_hosts()
->set_value(1);
// Set success_rate_request_volume to 4 when we only send 3 RPC in the // Set success_rate_request_volume to 4 when we only send 3 RPC in the
// interval. // interval.
// Note this parameter is the only difference between this test and // Note this parameter is the only difference between this test and
// SuccessRateEjectionAndUnejection (ejection portion, value set to 1) and // SuccessRateEjectionAndUnejection (ejection portion, value set to 1) and
// this one value changes means the difference between not ejecting in this // this one value changes means the difference between not ejecting in this
// test and ejecting in the other test. // test and ejecting in the other test.
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_request_volume()->set_value(4);
->mutable_success_rate_request_volume()
->set_value(4);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -527,23 +482,15 @@ TEST_P(OutlierDetectionTest, FailurePercentageEjectionAndUnejection) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
auto* base_time = outlier_detection->mutable_interval());
cluster.mutable_outlier_detection()->mutable_base_ejection_time(); SetProtoDuration(grpc_core::Duration::Seconds(3),
base_time->set_seconds(3 * grpc_test_slowdown_factor()); outlier_detection->mutable_base_ejection_time());
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
->mutable_failure_percentage_threshold() outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
->set_value(0); outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -606,20 +553,13 @@ TEST_P(OutlierDetectionTest, FailurePercentageMaxPercentage) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection() outlier_detection->mutable_interval());
->mutable_failure_percentage_threshold() outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
->set_value(0); outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
->mutable_enforcing_failure_percentage() outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -707,28 +647,20 @@ TEST_P(OutlierDetectionTest, FailurePercentageThreshold) {
CreateAndStartBackends(2); CreateAndStartBackends(2);
auto cluster = default_cluster_; auto cluster = default_cluster_;
cluster.set_lb_policy(Cluster::RING_HASH); cluster.set_lb_policy(Cluster::RING_HASH);
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
auto* base_time = outlier_detection->mutable_interval());
cluster.mutable_outlier_detection()->mutable_base_ejection_time(); SetProtoDuration(grpc_core::Duration::Seconds(1),
base_time->set_seconds(1 * grpc_test_slowdown_factor()); outlier_detection->mutable_base_ejection_time());
// Setup outlier failure percentage parameter to 50 // Setup outlier failure percentage parameter to 50
// Note this parameter is the only difference between this test and // Note this parameter is the only difference between this test and
// FailurePercentageEjectionAndUnejection (ejection portion, value set to 0) // FailurePercentageEjectionAndUnejection (ejection portion, value set to 0)
// and this one value changes means the difference between not ejecting in // and this one value changes means the difference between not ejecting in
// this test and ejecting in the other test. // this test and ejecting in the other test.
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_threshold()->set_value(50);
->mutable_failure_percentage_threshold() outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
->set_value(50); outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -778,28 +710,20 @@ TEST_P(OutlierDetectionTest, FailurePercentageEnforcementPercentage) {
CreateAndStartBackends(2); CreateAndStartBackends(2);
auto cluster = default_cluster_; auto cluster = default_cluster_;
cluster.set_lb_policy(Cluster::RING_HASH); cluster.set_lb_policy(Cluster::RING_HASH);
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
auto* base_time = outlier_detection->mutable_interval());
cluster.mutable_outlier_detection()->mutable_base_ejection_time(); SetProtoDuration(grpc_core::Duration::Seconds(1),
base_time->set_seconds(1 * grpc_test_slowdown_factor()); outlier_detection->mutable_base_ejection_time());
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
->mutable_failure_percentage_threshold()
->set_value(0);
// Setting enforcing_success_rate to 0 to ensure we will never eject. // Setting enforcing_success_rate to 0 to ensure we will never eject.
// Note this parameter is the only difference between this test and // Note this parameter is the only difference between this test and
// FailurePercentageEjectionAndUnejection (ejection portion, value set to 100) // FailurePercentageEjectionAndUnejection (ejection portion, value set to 100)
// and this one value changes means the difference between guaranteed not // and this one value changes means the difference between guaranteed not
// ejecting in this test and guaranteed ejecting in the other test. // ejecting in this test and guaranteed ejecting in the other test.
cluster.mutable_outlier_detection() outlier_detection->mutable_enforcing_failure_percentage()->set_value(0);
->mutable_enforcing_failure_percentage() outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
->set_value(0); outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -851,14 +775,11 @@ TEST_P(OutlierDetectionTest, FailurePercentageMinimumHosts) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection() outlier_detection->mutable_interval());
->mutable_failure_percentage_threshold() outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
->set_value(0); outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
// Set failure_percentage_minimum_hosts to 3 when we only have 2 backends // Set failure_percentage_minimum_hosts to 3 when we only have 2 backends
// Note this parameter is the only difference between this test and // Note this parameter is the only difference between this test and
// FailurePercentageEjectionAndUnejection (ejection portion, value set to 1) // FailurePercentageEjectionAndUnejection (ejection portion, value set to 1)
@ -922,26 +843,19 @@ TEST_P(OutlierDetectionTest, FailurePercentageRequestVolume) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection() outlier_detection->mutable_interval());
->mutable_failure_percentage_threshold() outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
->set_value(0); outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
// Set failure_percentage_request_volume to 4 when we only send 3 RPC in the // Set failure_percentage_request_volume to 4 when we only send 3 RPC in the
// interval. // interval.
// // Note this parameter is the only difference between this test and // // Note this parameter is the only difference between this test and
// FailurePercentageEjectionAndUnejection (ejection portion, value set to 1) // FailurePercentageEjectionAndUnejection (ejection portion, value set to 1)
// and this one value changes means the difference between not ejecting in // and this one value changes means the difference between not ejecting in
// this test and ejecting in the other test. // this test and ejecting in the other test.
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_request_volume()->set_value(4);
->mutable_failure_percentage_request_volume()
->set_value(4);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -995,37 +909,20 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection() outlier_detection->mutable_interval());
->mutable_max_ejection_percent() outlier_detection->mutable_max_ejection_percent()->set_value(50);
->set_value(50);
// This stdev of 500 will ensure the number of ok RPC and error RPC we send // This stdev of 500 will ensure the number of ok RPC and error RPC we send
// will make 1 outlier out of the 4 backends. // will make 1 outlier out of the 4 backends.
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_stdev_factor()->set_value(500);
->mutable_success_rate_stdev_factor() outlier_detection->mutable_enforcing_success_rate()->set_value(100);
->set_value(500); outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_request_volume()->set_value(1);
->mutable_enforcing_success_rate() outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
->set_value(100); outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
->mutable_success_rate_minimum_hosts() outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -1120,7 +1017,7 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentage) {
} }
// Tests SuccessRate and FailurePercentage both unconfigured; // Tests SuccessRate and FailurePercentage both unconfigured;
// This is the case where according to the RFC we need to instruct the picker // This is the case where according to the gRFC we need to instruct the picker
// not to do counting or even start the timer. The result of not counting is // not to do counting or even start the timer. The result of not counting is
// that there will be no ejection taking place since we can't do any // that there will be no ejection taking place since we can't do any
// calculations. // calculations.
@ -1130,14 +1027,11 @@ TEST_P(OutlierDetectionTest, SuccessRateAndFailurePercentageBothDisabled) {
CreateAndStartBackends(2); CreateAndStartBackends(2);
auto cluster = default_cluster_; auto cluster = default_cluster_;
cluster.set_lb_policy(Cluster::RING_HASH); cluster.set_lb_policy(Cluster::RING_HASH);
// Setup outlier failure percentage parameters. auto* outlier_detection = cluster.mutable_outlier_detection();
// Any failure will cause an potential ejection with the probability of 100% SetProtoDuration(grpc_core::Duration::Seconds(1),
// (to eliminate flakiness of the test). outlier_detection->mutable_interval());
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); SetProtoDuration(grpc_core::Duration::Seconds(1),
auto* base_time = outlier_detection->mutable_base_ejection_time());
cluster.mutable_outlier_detection()->mutable_base_ejection_time();
interval->set_seconds(1 * grpc_test_slowdown_factor());
base_time->set_seconds(1 * grpc_test_slowdown_factor());
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -1186,37 +1080,20 @@ TEST_P(OutlierDetectionTest,
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
cluster.mutable_outlier_detection() outlier_detection->mutable_interval());
->mutable_max_ejection_percent() outlier_detection->mutable_max_ejection_percent()->set_value(50);
->set_value(50);
// This stdev of 500 will ensure the number of ok RPC and error RPC we send // This stdev of 500 will ensure the number of ok RPC and error RPC we send
// will make 1 outlier out of the 4 backends. // will make 1 outlier out of the 4 backends.
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_stdev_factor()->set_value(500);
->mutable_success_rate_stdev_factor() outlier_detection->mutable_enforcing_success_rate()->set_value(100);
->set_value(500); outlier_detection->mutable_success_rate_minimum_hosts()->set_value(1);
cluster.mutable_outlier_detection() outlier_detection->mutable_success_rate_request_volume()->set_value(1);
->mutable_enforcing_success_rate() outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
->set_value(100); outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
->mutable_success_rate_minimum_hosts() outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_success_rate_request_volume()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_threshold()
->set_value(0);
cluster.mutable_outlier_detection()
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
@ -1293,23 +1170,15 @@ TEST_P(OutlierDetectionTest, DisableOutlierDetectionWhileAddressesAreEjected) {
// Setup outlier failure percentage parameters. // Setup outlier failure percentage parameters.
// Any failure will cause an potential ejection with the probability of 100% // Any failure will cause an potential ejection with the probability of 100%
// (to eliminate flakiness of the test). // (to eliminate flakiness of the test).
auto* interval = cluster.mutable_outlier_detection()->mutable_interval(); auto* outlier_detection = cluster.mutable_outlier_detection();
interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(grpc_core::Duration::Seconds(1),
auto* base_time = outlier_detection->mutable_interval());
cluster.mutable_outlier_detection()->mutable_base_ejection_time(); SetProtoDuration(grpc_core::Duration::Seconds(3),
base_time->set_seconds(3 * grpc_test_slowdown_factor()); outlier_detection->mutable_base_ejection_time());
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_threshold()->set_value(0);
->mutable_failure_percentage_threshold() outlier_detection->mutable_enforcing_failure_percentage()->set_value(100);
->set_value(0); outlier_detection->mutable_failure_percentage_minimum_hosts()->set_value(1);
cluster.mutable_outlier_detection() outlier_detection->mutable_failure_percentage_request_volume()->set_value(1);
->mutable_enforcing_failure_percentage()
->set_value(100);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_minimum_hosts()
->set_value(1);
cluster.mutable_outlier_detection()
->mutable_failure_percentage_request_volume()
->set_value(1);
balancer_->ads_service()->SetCdsResource(cluster); balancer_->ads_service()->SetCdsResource(cluster);
auto new_route_config = default_route_config_; auto new_route_config = default_route_config_;
auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0); auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);

@ -285,7 +285,8 @@ TEST_P(RingHashTest,
// Increase subchannel backoff time, so that subchannels stay in // Increase subchannel backoff time, so that subchannels stay in
// TRANSIENT_FAILURE for long enough to trigger potential problems. // TRANSIENT_FAILURE for long enough to trigger potential problems.
ChannelArguments channel_args; ChannelArguments channel_args;
channel_args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, 10000); channel_args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS,
10000 * grpc_test_slowdown_factor());
SetUpChannel(&channel_args); SetUpChannel(&channel_args);
// Start an RPC in the background. // Start an RPC in the background.
LongRunningRpc rpc; LongRunningRpc rpc;

@ -1882,12 +1882,12 @@ TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
} }
TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) { TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
const int64_t kTimeoutMillis = 500; const auto kTimeoutGrpcHeaderMax = grpc_core::Duration::Milliseconds(1500);
const int64_t kTimeoutNano = kTimeoutMillis * 1000000; const auto kTimeoutMaxStreamDuration =
const int64_t kTimeoutGrpcTimeoutHeaderMaxSecond = 1; grpc_core::Duration::Milliseconds(2500);
const int64_t kTimeoutMaxStreamDurationSecond = 2; const auto kTimeoutHttpMaxStreamDuration =
const int64_t kTimeoutHttpMaxStreamDurationSecond = 3; grpc_core::Duration::Milliseconds(3500);
const int64_t kTimeoutApplicationSecond = 4; const auto kTimeoutApplication = grpc_core::Duration::Milliseconds(4500);
const char* kNewCluster1Name = "new_cluster_1"; const char* kNewCluster1Name = "new_cluster_1";
const char* kNewEdsService1Name = "new_eds_service_name_1"; const char* kNewEdsService1Name = "new_eds_service_name_1";
const char* kNewCluster2Name = "new_cluster_2"; const char* kNewCluster2Name = "new_cluster_2";
@ -1928,11 +1928,10 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
listener.mutable_api_listener()->mutable_api_listener()->UnpackTo( listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
&http_connection_manager); &http_connection_manager);
// Set up HTTP max_stream_duration of 3.5 seconds // Set up HTTP max_stream_duration of 3.5 seconds
auto* duration = SetProtoDuration(
kTimeoutHttpMaxStreamDuration,
http_connection_manager.mutable_common_http_protocol_options() http_connection_manager.mutable_common_http_protocol_options()
->mutable_max_stream_duration(); ->mutable_max_stream_duration());
duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
duration->set_nanos(kTimeoutNano);
listener.mutable_api_listener()->mutable_api_listener()->PackFrom( listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
http_connection_manager); http_connection_manager);
// Construct route config. // Construct route config.
@ -1944,20 +1943,17 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
route1->mutable_route()->set_cluster(kNewCluster1Name); route1->mutable_route()->set_cluster(kNewCluster1Name);
auto* max_stream_duration = auto* max_stream_duration =
route1->mutable_route()->mutable_max_stream_duration(); route1->mutable_route()->mutable_max_stream_duration();
duration = max_stream_duration->mutable_max_stream_duration(); SetProtoDuration(kTimeoutMaxStreamDuration,
duration->set_seconds(kTimeoutMaxStreamDurationSecond); max_stream_duration->mutable_max_stream_duration());
duration->set_nanos(kTimeoutNano); SetProtoDuration(kTimeoutGrpcHeaderMax,
duration = max_stream_duration->mutable_grpc_timeout_header_max(); max_stream_duration->mutable_grpc_timeout_header_max());
duration->set_seconds(kTimeoutGrpcTimeoutHeaderMaxSecond);
duration->set_nanos(kTimeoutNano);
// route 2: Set max_stream_duration of 2.5 seconds // route 2: Set max_stream_duration of 2.5 seconds
auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes(); auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2"); route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
route2->mutable_route()->set_cluster(kNewCluster2Name); route2->mutable_route()->set_cluster(kNewCluster2Name);
max_stream_duration = route2->mutable_route()->mutable_max_stream_duration(); max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
duration = max_stream_duration->mutable_max_stream_duration(); SetProtoDuration(kTimeoutMaxStreamDuration,
duration->set_seconds(kTimeoutMaxStreamDurationSecond); max_stream_duration->mutable_max_stream_duration());
duration->set_nanos(kTimeoutNano);
// route 3: No timeout values in route configuration // route 3: No timeout values in route configuration
auto* route3 = new_route_config.mutable_virtual_hosts(0)->add_routes(); auto* route3 = new_route_config.mutable_virtual_hosts(0)->add_routes();
route3->mutable_match()->set_path("/grpc.testing.EchoTestService/Echo"); route3->mutable_match()->set_path("/grpc.testing.EchoTestService/Echo");
@ -1968,55 +1964,45 @@ TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
// Test grpc_timeout_header_max of 1.5 seconds applied // Test grpc_timeout_header_max of 1.5 seconds applied
grpc_core::Timestamp t0 = NowFromCycleCounter(); grpc_core::Timestamp t0 = NowFromCycleCounter();
grpc_core::Timestamp t1 = grpc_core::Timestamp t1 =
t0 + grpc_core::Duration::Seconds(kTimeoutGrpcTimeoutHeaderMaxSecond) + t0 + (kTimeoutGrpcHeaderMax * grpc_test_slowdown_factor());
grpc_core::Duration::Milliseconds(kTimeoutMillis);
grpc_core::Timestamp t2 = grpc_core::Timestamp t2 =
t0 + grpc_core::Duration::Seconds(kTimeoutMaxStreamDurationSecond) + t0 + (kTimeoutMaxStreamDuration * grpc_test_slowdown_factor());
grpc_core::Duration::Milliseconds(kTimeoutMillis); CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
CheckRpcSendFailure( "Deadline Exceeded",
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", RpcOptions()
RpcOptions() .set_rpc_service(SERVICE_ECHO1)
.set_rpc_service(SERVICE_ECHO1) .set_rpc_method(METHOD_ECHO1)
.set_rpc_method(METHOD_ECHO1) .set_wait_for_ready(true)
.set_wait_for_ready(true) .set_timeout(kTimeoutApplication));
.set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond)
.millis()));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2)); EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
// Test max_stream_duration of 2.5 seconds applied // Test max_stream_duration of 2.5 seconds applied
t0 = NowFromCycleCounter(); t0 = NowFromCycleCounter();
t1 = t0 + grpc_core::Duration::Seconds(kTimeoutMaxStreamDurationSecond) + t1 = t0 + (kTimeoutMaxStreamDuration * grpc_test_slowdown_factor());
grpc_core::Duration::Milliseconds(kTimeoutMillis); t2 = t0 + (kTimeoutHttpMaxStreamDuration * grpc_test_slowdown_factor());
t2 = t0 + grpc_core::Duration::Seconds(kTimeoutHttpMaxStreamDurationSecond) + CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
grpc_core::Duration::Milliseconds(kTimeoutMillis); "Deadline Exceeded",
CheckRpcSendFailure( RpcOptions()
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", .set_rpc_service(SERVICE_ECHO2)
RpcOptions() .set_rpc_method(METHOD_ECHO2)
.set_rpc_service(SERVICE_ECHO2) .set_wait_for_ready(true)
.set_rpc_method(METHOD_ECHO2) .set_timeout(kTimeoutApplication));
.set_wait_for_ready(true)
.set_timeout_ms(
grpc_core::Duration::Seconds(kTimeoutApplicationSecond)
.millis()));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2)); EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
// Test http_stream_duration of 3.5 seconds applied // Test http_stream_duration of 3.5 seconds applied
t0 = NowFromCycleCounter(); t0 = NowFromCycleCounter();
t1 = t0 + grpc_core::Duration::Seconds(kTimeoutHttpMaxStreamDurationSecond) + t1 = t0 + (kTimeoutHttpMaxStreamDuration * grpc_test_slowdown_factor());
grpc_core::Duration::Milliseconds(kTimeoutMillis); t2 = t0 + (kTimeoutApplication * grpc_test_slowdown_factor());
t2 = t0 + grpc_core::Duration::Seconds(kTimeoutApplicationSecond) +
grpc_core::Duration::Milliseconds(kTimeoutMillis);
CheckRpcSendFailure( CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms( RpcOptions().set_wait_for_ready(true).set_timeout(kTimeoutApplication));
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis()));
EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2)); EXPECT_THAT(NowFromCycleCounter(), AdjustedClockInRange(t1, t2));
} }
TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) { TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit) {
const int64_t kTimeoutNano = 500000000; const auto kTimeoutMaxStreamDuration =
const int64_t kTimeoutMaxStreamDurationSecond = 2; grpc_core::Duration::Milliseconds(2500);
const int64_t kTimeoutHttpMaxStreamDurationSecond = 3; const auto kTimeoutHttpMaxStreamDuration =
const int64_t kTimeoutApplicationSecond = 4; grpc_core::Duration::Milliseconds(3500);
const auto kTimeoutApplication = grpc_core::Duration::Milliseconds(4500);
const char* kNewCluster1Name = "new_cluster_1"; const char* kNewCluster1Name = "new_cluster_1";
const char* kNewEdsService1Name = "new_eds_service_name_1"; const char* kNewEdsService1Name = "new_eds_service_name_1";
const char* kNewCluster2Name = "new_cluster_2"; const char* kNewCluster2Name = "new_cluster_2";
@ -2047,11 +2033,10 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
listener.mutable_api_listener()->mutable_api_listener()->UnpackTo( listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
&http_connection_manager); &http_connection_manager);
// Set up HTTP max_stream_duration of 3.5 seconds // Set up HTTP max_stream_duration of 3.5 seconds
auto* duration = SetProtoDuration(
kTimeoutHttpMaxStreamDuration,
http_connection_manager.mutable_common_http_protocol_options() http_connection_manager.mutable_common_http_protocol_options()
->mutable_max_stream_duration(); ->mutable_max_stream_duration());
duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
duration->set_nanos(kTimeoutNano);
listener.mutable_api_listener()->mutable_api_listener()->PackFrom( listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
http_connection_manager); http_connection_manager);
// Construct route config. // Construct route config.
@ -2063,20 +2048,17 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
route1->mutable_route()->set_cluster(kNewCluster1Name); route1->mutable_route()->set_cluster(kNewCluster1Name);
auto* max_stream_duration = auto* max_stream_duration =
route1->mutable_route()->mutable_max_stream_duration(); route1->mutable_route()->mutable_max_stream_duration();
duration = max_stream_duration->mutable_max_stream_duration(); SetProtoDuration(kTimeoutMaxStreamDuration,
duration->set_seconds(kTimeoutMaxStreamDurationSecond); max_stream_duration->mutable_max_stream_duration());
duration->set_nanos(kTimeoutNano); SetProtoDuration(grpc_core::Duration::Zero(),
duration = max_stream_duration->mutable_grpc_timeout_header_max(); max_stream_duration->mutable_grpc_timeout_header_max());
duration->set_seconds(0);
duration->set_nanos(0);
// route 2: Set max_stream_duration to 0 // route 2: Set max_stream_duration to 0
auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes(); auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2"); route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
route2->mutable_route()->set_cluster(kNewCluster2Name); route2->mutable_route()->set_cluster(kNewCluster2Name);
max_stream_duration = route2->mutable_route()->mutable_max_stream_duration(); max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
duration = max_stream_duration->mutable_max_stream_duration(); SetProtoDuration(grpc_core::Duration::Zero(),
duration->set_seconds(0); max_stream_duration->mutable_max_stream_duration());
duration->set_nanos(0);
// Set listener and route config. // Set listener and route config.
SetListenerAndRouteConfiguration(balancer_.get(), std::move(listener), SetListenerAndRouteConfiguration(balancer_.get(), std::move(listener),
new_route_config); new_route_config);
@ -2088,12 +2070,13 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
.set_rpc_service(SERVICE_ECHO1) .set_rpc_service(SERVICE_ECHO1)
.set_rpc_method(METHOD_ECHO1) .set_rpc_method(METHOD_ECHO1)
.set_wait_for_ready(true) .set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000)); .set_timeout(kTimeoutApplication));
auto ellapsed_nano_seconds = auto elapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() - std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0); t0);
EXPECT_GT(ellapsed_nano_seconds.count(), EXPECT_GT(
kTimeoutApplicationSecond * 1000000000); elapsed_nano_seconds.count(),
(kTimeoutApplication * grpc_test_slowdown_factor()).millis() * 1000);
// Test application timeout is applied for route 2 // Test application timeout is applied for route 2
t0 = system_clock::now(); t0 = system_clock::now();
CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED,
@ -2102,15 +2085,16 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
.set_rpc_service(SERVICE_ECHO2) .set_rpc_service(SERVICE_ECHO2)
.set_rpc_method(METHOD_ECHO2) .set_rpc_method(METHOD_ECHO2)
.set_wait_for_ready(true) .set_wait_for_ready(true)
.set_timeout_ms(kTimeoutApplicationSecond * 1000)); .set_timeout(kTimeoutApplication));
ellapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>( elapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
system_clock::now() - t0); system_clock::now() - t0);
EXPECT_GT(ellapsed_nano_seconds.count(), EXPECT_GT(
kTimeoutApplicationSecond * 1000000000); elapsed_nano_seconds.count(),
(kTimeoutApplication * grpc_test_slowdown_factor()).millis() * 1000);
} }
TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) { TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit) {
const int64_t kTimeoutApplicationSecond = 4; const auto kTimeoutApplication = grpc_core::Duration::Milliseconds(4500);
// Populate new EDS resources. // Populate new EDS resources.
EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}}); EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
@ -2133,32 +2117,32 @@ TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) {
auto t0 = system_clock::now(); auto t0 = system_clock::now();
CheckRpcSendFailure( CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms( RpcOptions().set_wait_for_ready(true).set_timeout(kTimeoutApplication));
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis())); auto elapsed_nano_seconds =
auto ellapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() - std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0); t0);
EXPECT_GT(ellapsed_nano_seconds.count(), EXPECT_GT(
kTimeoutApplicationSecond * 1000000000); elapsed_nano_seconds.count(),
(kTimeoutApplication * grpc_test_slowdown_factor()).millis() * 1000);
} }
// Test to ensure application-specified deadline won't be affected when // Test to ensure application-specified deadline won't be affected when
// the xDS config does not specify a timeout. // the xDS config does not specify a timeout.
TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) { TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) {
const int64_t kTimeoutApplicationSecond = 4; const auto kTimeoutApplication = grpc_core::Duration::Milliseconds(4500);
// Populate new EDS resources. // Populate new EDS resources.
EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}}); EdsResourceArgs args({{"locality0", {MakeNonExistantEndpoint()}}});
balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
auto t0 = system_clock::now(); auto t0 = system_clock::now();
CheckRpcSendFailure( CheckRpcSendFailure(
DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded", DEBUG_LOCATION, StatusCode::DEADLINE_EXCEEDED, "Deadline Exceeded",
RpcOptions().set_wait_for_ready(true).set_timeout_ms( RpcOptions().set_wait_for_ready(true).set_timeout(kTimeoutApplication));
grpc_core::Duration::Seconds(kTimeoutApplicationSecond).millis())); auto elapsed_nano_seconds =
auto ellapsed_nano_seconds =
std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() - std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
t0); t0);
EXPECT_GT(ellapsed_nano_seconds.count(), EXPECT_GT(
kTimeoutApplicationSecond * 1000000000); elapsed_nano_seconds.count(),
(kTimeoutApplication * grpc_test_slowdown_factor()).millis() * 1000);
} }
TEST_P(LdsRdsTest, XdsRetryPolicyNumRetries) { TEST_P(LdsRdsTest, XdsRetryPolicyNumRetries) {
@ -2252,11 +2236,10 @@ TEST_P(LdsRdsTest, XdsRetryPolicyLongBackOff) {
"5xx,cancelled,deadline-exceeded,internal,resource-exhausted," "5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
"unavailable"); "unavailable");
retry_policy->mutable_num_retries()->set_value(kNumRetries); retry_policy->mutable_num_retries()->set_value(kNumRetries);
auto base_interval =
retry_policy->mutable_retry_back_off()->mutable_base_interval();
// Set backoff to 1 second, 1/2 of rpc timeout of 2 second. // Set backoff to 1 second, 1/2 of rpc timeout of 2 second.
base_interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(
base_interval->set_nanos(0); grpc_core::Duration::Seconds(1),
retry_policy->mutable_retry_back_off()->mutable_base_interval());
SetRouteConfiguration(balancer_.get(), new_route_config); SetRouteConfiguration(balancer_.get(), new_route_config);
// No need to set max interval and just let it be the default of 10x of base. // No need to set max interval and just let it be the default of 10x of base.
// We expect 1 retry before the RPC times out with DEADLINE_EXCEEDED. // We expect 1 retry before the RPC times out with DEADLINE_EXCEEDED.
@ -2285,19 +2268,17 @@ TEST_P(LdsRdsTest, XdsRetryPolicyMaxBackOff) {
"5xx,cancelled,deadline-exceeded,internal,resource-exhausted," "5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
"unavailable"); "unavailable");
retry_policy->mutable_num_retries()->set_value(kNumRetries); retry_policy->mutable_num_retries()->set_value(kNumRetries);
auto base_interval =
retry_policy->mutable_retry_back_off()->mutable_base_interval();
// Set backoff to 1 second. // Set backoff to 1 second.
base_interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(
base_interval->set_nanos(0); grpc_core::Duration::Seconds(1),
auto max_interval = retry_policy->mutable_retry_back_off()->mutable_base_interval());
retry_policy->mutable_retry_back_off()->mutable_max_interval();
// Set max interval to be the same as base, so 2 retries will take 2 seconds // Set max interval to be the same as base, so 2 retries will take 2 seconds
// and both retries will take place before the 2.5 seconds rpc timeout. // and both retries will take place before the 2.5 seconds rpc timeout.
// Tested to ensure if max is not set, this test will be the same as // Tested to ensure if max is not set, this test will be the same as
// XdsRetryPolicyLongBackOff and we will only see 1 retry in that case. // XdsRetryPolicyLongBackOff and we will only see 1 retry in that case.
max_interval->set_seconds(1 * grpc_test_slowdown_factor()); SetProtoDuration(
max_interval->set_nanos(0); grpc_core::Duration::Seconds(1),
retry_policy->mutable_retry_back_off()->mutable_max_interval());
SetRouteConfiguration(balancer_.get(), new_route_config); SetRouteConfiguration(balancer_.get(), new_route_config);
// Send an initial RPC to make sure we get connected (we don't want // Send an initial RPC to make sure we get connected (we don't want
// the channel startup time to affect the retry timing). // the channel startup time to affect the retry timing).
@ -2400,10 +2381,9 @@ TEST_P(LdsRdsTest, XdsRetryPolicyRetryBackOffMissingBaseInterval) {
retry_policy->set_retry_on("deadline-exceeded"); retry_policy->set_retry_on("deadline-exceeded");
retry_policy->mutable_num_retries()->set_value(1); retry_policy->mutable_num_retries()->set_value(1);
// RetryBackoff is there but base interval is missing. // RetryBackoff is there but base interval is missing.
auto max_interval = SetProtoDuration(
retry_policy->mutable_retry_back_off()->mutable_max_interval(); grpc_core::Duration::Milliseconds(250),
max_interval->set_seconds(0); retry_policy->mutable_retry_back_off()->mutable_max_interval());
max_interval->set_nanos(250000000);
SetRouteConfiguration(balancer_.get(), new_route_config); SetRouteConfiguration(balancer_.get(), new_route_config);
const auto response_state = WaitForRdsNack(DEBUG_LOCATION); const auto response_state = WaitForRdsNack(DEBUG_LOCATION);
ASSERT_TRUE(response_state.has_value()) << "timed out waiting for NACK"; ASSERT_TRUE(response_state.has_value()) << "timed out waiting for NACK";

@ -853,7 +853,8 @@ class LrsServiceImpl : public std::enable_shared_from_this<LrsServiceImpl> {
} }
} }
response.mutable_load_reporting_interval()->set_seconds( response.mutable_load_reporting_interval()->set_seconds(
parent_->client_load_reporting_interval_seconds_); parent_->client_load_reporting_interval_seconds_ *
grpc_test_slowdown_factor());
stream->Write(response); stream->Write(response);
CountedService<typename RpcApi::Service>::IncreaseResponseCount(); CountedService<typename RpcApi::Service>::IncreaseResponseCount();
// Wait for report. // Wait for report.

Loading…
Cancel
Save