|
|
|
|
@ -17,16 +17,27 @@ |
|
|
|
|
#include <stddef.h> |
|
|
|
|
#include <stdint.h> |
|
|
|
|
|
|
|
|
|
#include <algorithm> |
|
|
|
|
#include <array> |
|
|
|
|
#include <chrono> |
|
|
|
|
#include <memory> |
|
|
|
|
#include <ratio> |
|
|
|
|
#include <string> |
|
|
|
|
#include <utility> |
|
|
|
|
#include <vector> |
|
|
|
|
|
|
|
|
|
#include "absl/status/status.h" |
|
|
|
|
#include "absl/strings/string_view.h" |
|
|
|
|
#include "absl/types/optional.h" |
|
|
|
|
#include "gtest/gtest.h" |
|
|
|
|
|
|
|
|
|
#include <grpc/event_engine/event_engine.h> |
|
|
|
|
#include <grpc/grpc.h> |
|
|
|
|
#include <grpc/support/json.h> |
|
|
|
|
#include <grpc/support/log.h> |
|
|
|
|
|
|
|
|
|
#include "src/core/ext/filters/client_channel/lb_policy/backend_metric_data.h" |
|
|
|
|
#include "src/core/lib/channel/channel_args.h" |
|
|
|
|
#include "src/core/lib/gprpp/orphanable.h" |
|
|
|
|
#include "src/core/lib/gprpp/ref_counted_ptr.h" |
|
|
|
|
#include "src/core/lib/gprpp/time.h" |
|
|
|
|
@ -39,7 +50,7 @@ namespace grpc_core { |
|
|
|
|
namespace testing { |
|
|
|
|
namespace { |
|
|
|
|
|
|
|
|
|
class OutlierDetectionTest : public LoadBalancingPolicyTest { |
|
|
|
|
class OutlierDetectionTest : public TimeAwareLoadBalancingPolicyTest { |
|
|
|
|
protected: |
|
|
|
|
class ConfigBuilder { |
|
|
|
|
public: |
|
|
|
|
@ -137,7 +148,34 @@ class OutlierDetectionTest : public LoadBalancingPolicyTest { |
|
|
|
|
OutlierDetectionTest() |
|
|
|
|
: lb_policy_(MakeLbPolicy("outlier_detection_experimental")) {} |
|
|
|
|
|
|
|
|
|
absl::optional<std::string> DoPickWithFailedCall( |
|
|
|
|
LoadBalancingPolicy::SubchannelPicker* picker) { |
|
|
|
|
std::unique_ptr<LoadBalancingPolicy::SubchannelCallTrackerInterface> |
|
|
|
|
subchannel_call_tracker; |
|
|
|
|
auto address = ExpectPickComplete(picker, {}, &subchannel_call_tracker); |
|
|
|
|
if (address.has_value()) { |
|
|
|
|
subchannel_call_tracker->Start(); |
|
|
|
|
FakeMetadata metadata({}); |
|
|
|
|
FakeBackendMetricAccessor backend_metric_accessor({}); |
|
|
|
|
LoadBalancingPolicy::SubchannelCallTrackerInterface::FinishArgs args = { |
|
|
|
|
*address, absl::UnavailableError("uh oh"), &metadata, |
|
|
|
|
&backend_metric_accessor}; |
|
|
|
|
subchannel_call_tracker->Finish(args); |
|
|
|
|
} |
|
|
|
|
return address; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void CheckExpectedTimerDuration( |
|
|
|
|
grpc_event_engine::experimental::EventEngine::Duration duration) |
|
|
|
|
override { |
|
|
|
|
EXPECT_EQ(duration, expected_internal_) |
|
|
|
|
<< "Expected: " << expected_internal_.count() << "ns" |
|
|
|
|
<< "\n Actual: " << duration.count() << "ns"; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
OrphanablePtr<LoadBalancingPolicy> lb_policy_; |
|
|
|
|
grpc_event_engine::experimental::EventEngine::Duration expected_internal_ = |
|
|
|
|
std::chrono::seconds(10); |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
TEST_F(OutlierDetectionTest, Basic) { |
|
|
|
|
@ -168,6 +206,93 @@ TEST_F(OutlierDetectionTest, Basic) { |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
TEST_F(OutlierDetectionTest, FailurePercentage) { |
|
|
|
|
constexpr std::array<absl::string_view, 3> kAddresses = { |
|
|
|
|
"ipv4:127.0.0.1:440", "ipv4:127.0.0.1:441", "ipv4:127.0.0.1:442"}; |
|
|
|
|
// Send initial update.
|
|
|
|
|
absl::Status status = ApplyUpdate( |
|
|
|
|
BuildUpdate(kAddresses, ConfigBuilder() |
|
|
|
|
.SetFailurePercentageThreshold(1) |
|
|
|
|
.SetFailurePercentageMinimumHosts(1) |
|
|
|
|
.SetFailurePercentageRequestVolume(1) |
|
|
|
|
.Build()), |
|
|
|
|
lb_policy_.get()); |
|
|
|
|
EXPECT_TRUE(status.ok()) << status; |
|
|
|
|
// Expect normal startup.
|
|
|
|
|
auto picker = ExpectRoundRobinStartup(kAddresses); |
|
|
|
|
ASSERT_NE(picker, nullptr); |
|
|
|
|
gpr_log(GPR_INFO, "### RR startup complete"); |
|
|
|
|
// Do a pick and report a failed call.
|
|
|
|
|
auto address = DoPickWithFailedCall(picker.get()); |
|
|
|
|
ASSERT_TRUE(address.has_value()); |
|
|
|
|
gpr_log(GPR_INFO, "### failed RPC on %s", address->c_str()); |
|
|
|
|
// Advance time and run the timer callback to trigger ejection.
|
|
|
|
|
time_cache_.IncrementBy(Duration::Seconds(10)); |
|
|
|
|
RunTimerCallback(); |
|
|
|
|
gpr_log(GPR_INFO, "### ejection complete"); |
|
|
|
|
// Expect a re-resolution request.
|
|
|
|
|
ExpectReresolutionRequest(); |
|
|
|
|
// Expect a picker update.
|
|
|
|
|
std::vector<absl::string_view> remaining_addresses; |
|
|
|
|
for (const auto& addr : kAddresses) { |
|
|
|
|
if (addr != *address) remaining_addresses.push_back(addr); |
|
|
|
|
} |
|
|
|
|
picker = WaitForRoundRobinListChange(kAddresses, remaining_addresses); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
TEST_F(OutlierDetectionTest, FailurePercentageWithPickFirst) { |
|
|
|
|
constexpr std::array<absl::string_view, 3> kAddresses = { |
|
|
|
|
"ipv4:127.0.0.1:440", "ipv4:127.0.0.1:441", "ipv4:127.0.0.1:442"}; |
|
|
|
|
// Send initial update.
|
|
|
|
|
absl::Status status = ApplyUpdate( |
|
|
|
|
BuildUpdate(kAddresses, |
|
|
|
|
ConfigBuilder() |
|
|
|
|
.SetFailurePercentageThreshold(1) |
|
|
|
|
.SetFailurePercentageMinimumHosts(1) |
|
|
|
|
.SetFailurePercentageRequestVolume(1) |
|
|
|
|
.SetChildPolicy({{"pick_first", Json::FromObject({})}}) |
|
|
|
|
.Build()), |
|
|
|
|
lb_policy_.get()); |
|
|
|
|
EXPECT_TRUE(status.ok()) << status; |
|
|
|
|
// LB policy should have created a subchannel for the first address with
|
|
|
|
|
// the GRPC_ARG_INHIBIT_HEALTH_CHECKING channel arg.
|
|
|
|
|
auto* subchannel = FindSubchannel( |
|
|
|
|
kAddresses[0], ChannelArgs().Set(GRPC_ARG_INHIBIT_HEALTH_CHECKING, true)); |
|
|
|
|
ASSERT_NE(subchannel, nullptr); |
|
|
|
|
// When the LB policy receives the subchannel's initial connectivity
|
|
|
|
|
// state notification (IDLE), it will request a connection.
|
|
|
|
|
EXPECT_TRUE(subchannel->ConnectionRequested()); |
|
|
|
|
// This causes the subchannel to start to connect, so it reports CONNECTING.
|
|
|
|
|
subchannel->SetConnectivityState(GRPC_CHANNEL_CONNECTING); |
|
|
|
|
// LB policy should have reported CONNECTING state.
|
|
|
|
|
ExpectConnectingUpdate(); |
|
|
|
|
// When the subchannel becomes connected, it reports READY.
|
|
|
|
|
subchannel->SetConnectivityState(GRPC_CHANNEL_READY); |
|
|
|
|
// The LB policy will report CONNECTING some number of times (doesn't
|
|
|
|
|
// matter how many) and then report READY.
|
|
|
|
|
auto picker = WaitForConnected(); |
|
|
|
|
ASSERT_NE(picker, nullptr); |
|
|
|
|
// Picker should return the same subchannel repeatedly.
|
|
|
|
|
for (size_t i = 0; i < 3; ++i) { |
|
|
|
|
EXPECT_EQ(ExpectPickComplete(picker.get()), kAddresses[0]); |
|
|
|
|
} |
|
|
|
|
gpr_log(GPR_INFO, "### PF startup complete"); |
|
|
|
|
// Now have an RPC to that subchannel fail.
|
|
|
|
|
auto address = DoPickWithFailedCall(picker.get()); |
|
|
|
|
ASSERT_TRUE(address.has_value()); |
|
|
|
|
gpr_log(GPR_INFO, "### failed RPC on %s", address->c_str()); |
|
|
|
|
// Advance time and run the timer callback to trigger ejection.
|
|
|
|
|
time_cache_.IncrementBy(Duration::Seconds(10)); |
|
|
|
|
RunTimerCallback(); |
|
|
|
|
gpr_log(GPR_INFO, "### ejection complete"); |
|
|
|
|
// Expect a re-resolution request.
|
|
|
|
|
ExpectReresolutionRequest(); |
|
|
|
|
// The pick_first policy should report IDLE with a queuing picker.
|
|
|
|
|
ExpectStateAndQueuingPicker(GRPC_CHANNEL_IDLE); |
|
|
|
|
// The queued pick should have triggered a reconnection attempt.
|
|
|
|
|
EXPECT_TRUE(subchannel->ConnectionRequested()); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
} // namespace
|
|
|
|
|
} // namespace testing
|
|
|
|
|
} // namespace grpc_core
|
|
|
|
|
|
Mark D. Roth
2 years ago
committed by
GitHub