|
|
|
|
@ -22,7 +22,9 @@ |
|
|
|
|
#include <numeric> |
|
|
|
|
#include <set> |
|
|
|
|
#include <sstream> |
|
|
|
|
#include <string> |
|
|
|
|
#include <thread> |
|
|
|
|
#include <vector> |
|
|
|
|
|
|
|
|
|
#include <grpc/grpc.h> |
|
|
|
|
#include <grpc/support/alloc.h> |
|
|
|
|
@ -296,8 +298,9 @@ class ClientStats { |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
// Converts from proto message class.
|
|
|
|
|
ClientStats(const ClusterStats& cluster_stats) |
|
|
|
|
: total_dropped_requests_(cluster_stats.total_dropped_requests()) { |
|
|
|
|
explicit ClientStats(const ClusterStats& cluster_stats) |
|
|
|
|
: cluster_name_(cluster_stats.cluster_name()), |
|
|
|
|
total_dropped_requests_(cluster_stats.total_dropped_requests()) { |
|
|
|
|
for (const auto& input_locality_stats : |
|
|
|
|
cluster_stats.upstream_locality_stats()) { |
|
|
|
|
locality_stats_.emplace(input_locality_stats.locality().sub_zone(), |
|
|
|
|
@ -310,6 +313,11 @@ class ClientStats { |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
const std::string& cluster_name() const { return cluster_name_; } |
|
|
|
|
|
|
|
|
|
const std::map<grpc::string, LocalityStats>& locality_stats() const { |
|
|
|
|
return locality_stats_; |
|
|
|
|
} |
|
|
|
|
uint64_t total_successful_requests() const { |
|
|
|
|
uint64_t sum = 0; |
|
|
|
|
for (auto& p : locality_stats_) { |
|
|
|
|
@ -338,7 +346,9 @@ class ClientStats { |
|
|
|
|
} |
|
|
|
|
return sum; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
uint64_t total_dropped_requests() const { return total_dropped_requests_; } |
|
|
|
|
|
|
|
|
|
uint64_t dropped_requests(const grpc::string& category) const { |
|
|
|
|
auto iter = dropped_requests_.find(category); |
|
|
|
|
GPR_ASSERT(iter != dropped_requests_.end()); |
|
|
|
|
@ -346,6 +356,7 @@ class ClientStats { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private: |
|
|
|
|
std::string cluster_name_; |
|
|
|
|
std::map<grpc::string, LocalityStats> locality_stats_; |
|
|
|
|
uint64_t total_dropped_requests_; |
|
|
|
|
std::map<grpc::string, uint64_t> dropped_requests_; |
|
|
|
|
@ -391,7 +402,6 @@ class AdsServiceImpl : public AggregatedDiscoveryService::Service, |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
using Stream = ServerReaderWriter<DiscoveryResponse, DiscoveryRequest>; |
|
|
|
|
using ResponseDelayPair = std::pair<DiscoveryResponse, int>; |
|
|
|
|
|
|
|
|
|
// A queue of resource type/name pairs that have changed since the client
|
|
|
|
|
// subscribed to them.
|
|
|
|
|
@ -933,60 +943,62 @@ class LrsServiceImpl : public LrsService, |
|
|
|
|
|
|
|
|
|
explicit LrsServiceImpl(int client_load_reporting_interval_seconds) |
|
|
|
|
: client_load_reporting_interval_seconds_( |
|
|
|
|
client_load_reporting_interval_seconds) {} |
|
|
|
|
client_load_reporting_interval_seconds), |
|
|
|
|
cluster_names_({kDefaultResourceName}) {} |
|
|
|
|
|
|
|
|
|
Status StreamLoadStats(ServerContext* /*context*/, Stream* stream) override { |
|
|
|
|
gpr_log(GPR_INFO, "LRS[%p]: StreamLoadStats starts", this); |
|
|
|
|
GPR_ASSERT(client_load_reporting_interval_seconds_ > 0); |
|
|
|
|
// Take a reference of the LrsServiceImpl object, reference will go
|
|
|
|
|
// out of scope after this method exits.
|
|
|
|
|
std::shared_ptr<LrsServiceImpl> lrs_service_impl = shared_from_this(); |
|
|
|
|
// Read request.
|
|
|
|
|
// Read initial request.
|
|
|
|
|
LoadStatsRequest request; |
|
|
|
|
if (stream->Read(&request)) { |
|
|
|
|
if (client_load_reporting_interval_seconds_ > 0) { |
|
|
|
|
IncreaseRequestCount(); |
|
|
|
|
// Send response.
|
|
|
|
|
LoadStatsResponse response; |
|
|
|
|
std::string server_name; |
|
|
|
|
auto it = request.node().metadata().fields().find( |
|
|
|
|
"PROXYLESS_CLIENT_HOSTNAME"); |
|
|
|
|
if (it != request.node().metadata().fields().end()) { |
|
|
|
|
server_name = it->second.string_value(); |
|
|
|
|
} |
|
|
|
|
GPR_ASSERT(server_name != ""); |
|
|
|
|
response.add_clusters(server_name); |
|
|
|
|
response.mutable_load_reporting_interval()->set_seconds( |
|
|
|
|
client_load_reporting_interval_seconds_); |
|
|
|
|
stream->Write(response); |
|
|
|
|
IncreaseResponseCount(); |
|
|
|
|
// Wait for report.
|
|
|
|
|
request.Clear(); |
|
|
|
|
if (stream->Read(&request)) { |
|
|
|
|
gpr_log(GPR_INFO, "LRS[%p]: received client load report message '%s'", |
|
|
|
|
this, request.DebugString().c_str()); |
|
|
|
|
GPR_ASSERT(request.cluster_stats().size() == 1); |
|
|
|
|
const ClusterStats& cluster_stats = request.cluster_stats()[0]; |
|
|
|
|
// We need to acquire the lock here in order to prevent the notify_one
|
|
|
|
|
// below from firing before its corresponding wait is executed.
|
|
|
|
|
grpc_core::MutexLock lock(&load_report_mu_); |
|
|
|
|
GPR_ASSERT(client_stats_ == nullptr); |
|
|
|
|
client_stats_.reset(new ClientStats(cluster_stats)); |
|
|
|
|
load_report_ready_ = true; |
|
|
|
|
load_report_cond_.Signal(); |
|
|
|
|
IncreaseRequestCount(); // Only for initial request.
|
|
|
|
|
// Verify server name set in metadata.
|
|
|
|
|
auto it = |
|
|
|
|
request.node().metadata().fields().find("PROXYLESS_CLIENT_HOSTNAME"); |
|
|
|
|
GPR_ASSERT(it != request.node().metadata().fields().end()); |
|
|
|
|
EXPECT_EQ(it->second.string_value(), kDefaultResourceName); |
|
|
|
|
// Send initial response.
|
|
|
|
|
LoadStatsResponse response; |
|
|
|
|
for (const std::string& cluster_name : cluster_names_) { |
|
|
|
|
response.add_clusters(cluster_name); |
|
|
|
|
} |
|
|
|
|
response.mutable_load_reporting_interval()->set_seconds( |
|
|
|
|
client_load_reporting_interval_seconds_); |
|
|
|
|
stream->Write(response); |
|
|
|
|
IncreaseResponseCount(); |
|
|
|
|
// Wait for report.
|
|
|
|
|
request.Clear(); |
|
|
|
|
while (stream->Read(&request)) { |
|
|
|
|
gpr_log(GPR_INFO, "LRS[%p]: received client load report message: %s", |
|
|
|
|
this, request.DebugString().c_str()); |
|
|
|
|
std::vector<ClientStats> stats; |
|
|
|
|
for (const auto& cluster_stats : request.cluster_stats()) { |
|
|
|
|
stats.emplace_back(cluster_stats); |
|
|
|
|
} |
|
|
|
|
grpc_core::MutexLock lock(&load_report_mu_); |
|
|
|
|
result_queue_.emplace_back(std::move(stats)); |
|
|
|
|
if (load_report_cond_ != nullptr) load_report_cond_->Signal(); |
|
|
|
|
} |
|
|
|
|
// Wait until notified done.
|
|
|
|
|
grpc_core::MutexLock lock(&lrs_mu_); |
|
|
|
|
lrs_cv_.WaitUntil(&lrs_mu_, [this] { return lrs_done; }); |
|
|
|
|
lrs_cv_.WaitUntil(&lrs_mu_, [this] { return lrs_done_; }); |
|
|
|
|
} |
|
|
|
|
gpr_log(GPR_INFO, "LRS[%p]: StreamLoadStats done", this); |
|
|
|
|
return Status::OK; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Must be called before the LRS call is started.
|
|
|
|
|
void set_cluster_names(const std::set<std::string>& cluster_names) { |
|
|
|
|
cluster_names_ = cluster_names; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void Start() { |
|
|
|
|
lrs_done = false; |
|
|
|
|
load_report_ready_ = false; |
|
|
|
|
client_stats_.reset(); |
|
|
|
|
lrs_done_ = false; |
|
|
|
|
result_queue_.clear(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void Shutdown() { |
|
|
|
|
@ -997,12 +1009,18 @@ class LrsServiceImpl : public LrsService, |
|
|
|
|
gpr_log(GPR_INFO, "LRS[%p]: shut down", this); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
ClientStats* WaitForLoadReport() { |
|
|
|
|
std::vector<ClientStats> WaitForLoadReport() { |
|
|
|
|
grpc_core::MutexLock lock(&load_report_mu_); |
|
|
|
|
load_report_cond_.WaitUntil(&load_report_mu_, |
|
|
|
|
[this] { return load_report_ready_; }); |
|
|
|
|
load_report_ready_ = false; |
|
|
|
|
return client_stats_.get(); |
|
|
|
|
grpc_core::CondVar cv; |
|
|
|
|
if (result_queue_.empty()) { |
|
|
|
|
load_report_cond_ = &cv; |
|
|
|
|
load_report_cond_->WaitUntil(&load_report_mu_, |
|
|
|
|
[this] { return !result_queue_.empty(); }); |
|
|
|
|
load_report_cond_ = nullptr; |
|
|
|
|
} |
|
|
|
|
std::vector<ClientStats> result = std::move(result_queue_.front()); |
|
|
|
|
result_queue_.pop_front(); |
|
|
|
|
return result; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void NotifyDoneWithLrsCall() { |
|
|
|
|
@ -1010,26 +1028,24 @@ class LrsServiceImpl : public LrsService, |
|
|
|
|
NotifyDoneWithLrsCallLocked(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private: |
|
|
|
|
void NotifyDoneWithLrsCallLocked() { |
|
|
|
|
if (!lrs_done) { |
|
|
|
|
lrs_done = true; |
|
|
|
|
if (!lrs_done_) { |
|
|
|
|
lrs_done_ = true; |
|
|
|
|
lrs_cv_.Broadcast(); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private: |
|
|
|
|
const int client_load_reporting_interval_seconds_; |
|
|
|
|
std::set<std::string> cluster_names_; |
|
|
|
|
|
|
|
|
|
grpc_core::CondVar lrs_cv_; |
|
|
|
|
// Protect lrs_done.
|
|
|
|
|
grpc_core::Mutex lrs_mu_; |
|
|
|
|
bool lrs_done = false; |
|
|
|
|
grpc_core::Mutex lrs_mu_; // Protects lrs_done_.
|
|
|
|
|
bool lrs_done_ = false; |
|
|
|
|
|
|
|
|
|
grpc_core::CondVar load_report_cond_; |
|
|
|
|
// Protect the members below.
|
|
|
|
|
grpc_core::Mutex load_report_mu_; |
|
|
|
|
std::unique_ptr<ClientStats> client_stats_; |
|
|
|
|
bool load_report_ready_ = false; |
|
|
|
|
grpc_core::Mutex load_report_mu_; // Protects the members below.
|
|
|
|
|
grpc_core::CondVar* load_report_cond_ = nullptr; |
|
|
|
|
std::deque<std::vector<ClientStats>> result_queue_; |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
class TestType { |
|
|
|
|
@ -1720,6 +1736,141 @@ TEST_P(XdsResolverOnlyTest, ClusterRemoved) { |
|
|
|
|
AdsServiceImpl::ACKED); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
class XdsResolverLoadReportingOnlyTest : public XdsEnd2endTest { |
|
|
|
|
public: |
|
|
|
|
XdsResolverLoadReportingOnlyTest() : XdsEnd2endTest(4, 1, 3) {} |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
// Tests load reporting when switching over from one cluster to another.
|
|
|
|
|
TEST_P(XdsResolverLoadReportingOnlyTest, ChangeClusters) { |
|
|
|
|
const char* kNewClusterName = "new_cluster_name"; |
|
|
|
|
balancers_[0]->lrs_service()->set_cluster_names( |
|
|
|
|
{kDefaultResourceName, kNewClusterName}); |
|
|
|
|
SetNextResolution({}); |
|
|
|
|
SetNextResolutionForLbChannelAllBalancers(); |
|
|
|
|
// cluster kDefaultResourceName -> locality0 -> backends 0 and 1
|
|
|
|
|
AdsServiceImpl::EdsResourceArgs args({ |
|
|
|
|
{"locality0", GetBackendPorts(0, 2)}, |
|
|
|
|
}); |
|
|
|
|
balancers_[0]->ads_service()->SetEdsResource( |
|
|
|
|
AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); |
|
|
|
|
// cluster kNewClusterName -> locality1 -> backends 2 and 3
|
|
|
|
|
AdsServiceImpl::EdsResourceArgs args2({ |
|
|
|
|
{"locality1", GetBackendPorts(2, 4)}, |
|
|
|
|
}); |
|
|
|
|
balancers_[0]->ads_service()->SetEdsResource( |
|
|
|
|
AdsServiceImpl::BuildEdsResource(args2, kNewClusterName), |
|
|
|
|
kNewClusterName); |
|
|
|
|
// CDS resource for kNewClusterName.
|
|
|
|
|
Cluster new_cluster = balancers_[0]->ads_service()->default_cluster(); |
|
|
|
|
new_cluster.set_name(kNewClusterName); |
|
|
|
|
balancers_[0]->ads_service()->SetCdsResource(new_cluster, kNewClusterName); |
|
|
|
|
// Wait for all backends to come online.
|
|
|
|
|
int num_ok = 0; |
|
|
|
|
int num_failure = 0; |
|
|
|
|
int num_drops = 0; |
|
|
|
|
std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(0, 2); |
|
|
|
|
// The load report received at the balancer should be correct.
|
|
|
|
|
std::vector<ClientStats> load_report = |
|
|
|
|
balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
EXPECT_THAT( |
|
|
|
|
load_report, |
|
|
|
|
::testing::ElementsAre(::testing::AllOf( |
|
|
|
|
::testing::Property(&ClientStats::cluster_name, kDefaultResourceName), |
|
|
|
|
::testing::Property( |
|
|
|
|
&ClientStats::locality_stats, |
|
|
|
|
::testing::ElementsAre(::testing::Pair( |
|
|
|
|
"locality0", |
|
|
|
|
::testing::AllOf( |
|
|
|
|
::testing::Field(&ClientStats::LocalityStats:: |
|
|
|
|
total_successful_requests, |
|
|
|
|
num_ok), |
|
|
|
|
::testing::Field(&ClientStats::LocalityStats:: |
|
|
|
|
total_requests_in_progress, |
|
|
|
|
0UL), |
|
|
|
|
::testing::Field( |
|
|
|
|
&ClientStats::LocalityStats::total_error_requests, |
|
|
|
|
num_failure), |
|
|
|
|
::testing::Field( |
|
|
|
|
&ClientStats::LocalityStats::total_issued_requests, |
|
|
|
|
num_failure + num_ok))))), |
|
|
|
|
::testing::Property(&ClientStats::total_dropped_requests, |
|
|
|
|
num_drops)))); |
|
|
|
|
// Change RDS resource to point to new cluster.
|
|
|
|
|
RouteConfiguration new_route_config = |
|
|
|
|
balancers_[0]->ads_service()->default_route_config(); |
|
|
|
|
new_route_config.mutable_virtual_hosts(0) |
|
|
|
|
->mutable_routes(0) |
|
|
|
|
->mutable_route() |
|
|
|
|
->set_cluster(kNewClusterName); |
|
|
|
|
Listener listener = |
|
|
|
|
balancers_[0]->ads_service()->BuildListener(new_route_config); |
|
|
|
|
balancers_[0]->ads_service()->SetLdsResource(listener, kDefaultResourceName); |
|
|
|
|
// Wait for all new backends to be used.
|
|
|
|
|
std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(2, 4); |
|
|
|
|
// The load report received at the balancer should be correct.
|
|
|
|
|
load_report = balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
EXPECT_THAT( |
|
|
|
|
load_report, |
|
|
|
|
::testing::ElementsAre( |
|
|
|
|
::testing::AllOf( |
|
|
|
|
::testing::Property(&ClientStats::cluster_name, |
|
|
|
|
kDefaultResourceName), |
|
|
|
|
::testing::Property( |
|
|
|
|
&ClientStats::locality_stats, |
|
|
|
|
::testing::ElementsAre(::testing::Pair( |
|
|
|
|
"locality0", |
|
|
|
|
::testing::AllOf( |
|
|
|
|
::testing::Field(&ClientStats::LocalityStats:: |
|
|
|
|
total_successful_requests, |
|
|
|
|
::testing::Lt(num_ok)), |
|
|
|
|
::testing::Field(&ClientStats::LocalityStats:: |
|
|
|
|
total_requests_in_progress, |
|
|
|
|
0UL), |
|
|
|
|
::testing::Field( |
|
|
|
|
&ClientStats::LocalityStats::total_error_requests, |
|
|
|
|
::testing::Le(num_failure)), |
|
|
|
|
::testing::Field( |
|
|
|
|
&ClientStats::LocalityStats:: |
|
|
|
|
total_issued_requests, |
|
|
|
|
::testing::Le(num_failure + num_ok)))))), |
|
|
|
|
::testing::Property(&ClientStats::total_dropped_requests, |
|
|
|
|
num_drops)), |
|
|
|
|
::testing::AllOf( |
|
|
|
|
::testing::Property(&ClientStats::cluster_name, kNewClusterName), |
|
|
|
|
::testing::Property( |
|
|
|
|
&ClientStats::locality_stats, |
|
|
|
|
::testing::ElementsAre(::testing::Pair( |
|
|
|
|
"locality1", |
|
|
|
|
::testing::AllOf( |
|
|
|
|
::testing::Field(&ClientStats::LocalityStats:: |
|
|
|
|
total_successful_requests, |
|
|
|
|
::testing::Le(num_ok)), |
|
|
|
|
::testing::Field(&ClientStats::LocalityStats:: |
|
|
|
|
total_requests_in_progress, |
|
|
|
|
0UL), |
|
|
|
|
::testing::Field( |
|
|
|
|
&ClientStats::LocalityStats::total_error_requests, |
|
|
|
|
::testing::Le(num_failure)), |
|
|
|
|
::testing::Field( |
|
|
|
|
&ClientStats::LocalityStats:: |
|
|
|
|
total_issued_requests, |
|
|
|
|
::testing::Le(num_failure + num_ok)))))), |
|
|
|
|
::testing::Property(&ClientStats::total_dropped_requests, |
|
|
|
|
num_drops)))); |
|
|
|
|
int total_ok = 0; |
|
|
|
|
int total_failure = 0; |
|
|
|
|
for (const ClientStats& client_stats : load_report) { |
|
|
|
|
total_ok += client_stats.total_successful_requests(); |
|
|
|
|
total_failure += client_stats.total_error_requests(); |
|
|
|
|
} |
|
|
|
|
EXPECT_EQ(total_ok, num_ok); |
|
|
|
|
EXPECT_EQ(total_failure, num_failure); |
|
|
|
|
// The LRS service got a single request, and sent a single response.
|
|
|
|
|
EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count()); |
|
|
|
|
EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count()); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
using SecureNamingTest = BasicTest; |
|
|
|
|
|
|
|
|
|
// Tests that secure naming check passes if target name is expected.
|
|
|
|
|
@ -3227,14 +3378,50 @@ TEST_P(ClientLoadReportingTest, Vanilla) { |
|
|
|
|
EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count()); |
|
|
|
|
EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count()); |
|
|
|
|
// The load report received at the balancer should be correct.
|
|
|
|
|
ClientStats* client_stats = balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
std::vector<ClientStats> load_report = |
|
|
|
|
balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
ASSERT_EQ(load_report.size(), 1UL); |
|
|
|
|
ClientStats& client_stats = load_report.front(); |
|
|
|
|
EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok, |
|
|
|
|
client_stats->total_successful_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats->total_requests_in_progress()); |
|
|
|
|
client_stats.total_successful_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats.total_requests_in_progress()); |
|
|
|
|
EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok, |
|
|
|
|
client_stats->total_issued_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats->total_error_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats->total_dropped_requests()); |
|
|
|
|
client_stats.total_issued_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats.total_error_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats.total_dropped_requests()); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Tests that we don't include stats for clusters that are not requested
|
|
|
|
|
// by the LRS server.
|
|
|
|
|
TEST_P(ClientLoadReportingTest, HonorsClustersRequestedByLrsServer) { |
|
|
|
|
balancers_[0]->lrs_service()->set_cluster_names({"bogus"}); |
|
|
|
|
SetNextResolution({}); |
|
|
|
|
SetNextResolutionForLbChannel({balancers_[0]->port()}); |
|
|
|
|
const size_t kNumRpcsPerAddress = 100; |
|
|
|
|
AdsServiceImpl::EdsResourceArgs args({ |
|
|
|
|
{"locality0", GetBackendPorts()}, |
|
|
|
|
}); |
|
|
|
|
balancers_[0]->ads_service()->SetEdsResource( |
|
|
|
|
AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); |
|
|
|
|
// Wait until all backends are ready.
|
|
|
|
|
int num_ok = 0; |
|
|
|
|
int num_failure = 0; |
|
|
|
|
int num_drops = 0; |
|
|
|
|
std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(); |
|
|
|
|
// Send kNumRpcsPerAddress RPCs per server.
|
|
|
|
|
CheckRpcSendOk(kNumRpcsPerAddress * num_backends_); |
|
|
|
|
// Each backend should have gotten 100 requests.
|
|
|
|
|
for (size_t i = 0; i < backends_.size(); ++i) { |
|
|
|
|
EXPECT_EQ(kNumRpcsPerAddress, |
|
|
|
|
backends_[i]->backend_service()->request_count()); |
|
|
|
|
} |
|
|
|
|
// The LRS service got a single request, and sent a single response.
|
|
|
|
|
EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count()); |
|
|
|
|
EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count()); |
|
|
|
|
// The load report received at the balancer should be correct.
|
|
|
|
|
std::vector<ClientStats> load_report = |
|
|
|
|
balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
ASSERT_EQ(load_report.size(), 0UL); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Tests that if the balancer restarts, the client load report contains the
|
|
|
|
|
@ -3257,12 +3444,15 @@ TEST_P(ClientLoadReportingTest, BalancerRestart) { |
|
|
|
|
std::tie(num_ok, num_failure, num_drops) = |
|
|
|
|
WaitForAllBackends(/* start_index */ 0, |
|
|
|
|
/* stop_index */ kNumBackendsFirstPass); |
|
|
|
|
ClientStats* client_stats = balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
std::vector<ClientStats> load_report = |
|
|
|
|
balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
ASSERT_EQ(load_report.size(), 1UL); |
|
|
|
|
ClientStats client_stats = std::move(load_report.front()); |
|
|
|
|
EXPECT_EQ(static_cast<size_t>(num_ok), |
|
|
|
|
client_stats->total_successful_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats->total_requests_in_progress()); |
|
|
|
|
EXPECT_EQ(0U, client_stats->total_error_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats->total_dropped_requests()); |
|
|
|
|
client_stats.total_successful_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats.total_requests_in_progress()); |
|
|
|
|
EXPECT_EQ(0U, client_stats.total_error_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats.total_dropped_requests()); |
|
|
|
|
// Shut down the balancer.
|
|
|
|
|
balancers_[0]->Shutdown(); |
|
|
|
|
// We should continue using the last EDS response we received from the
|
|
|
|
|
@ -3294,11 +3484,13 @@ TEST_P(ClientLoadReportingTest, BalancerRestart) { |
|
|
|
|
CheckRpcSendOk(kNumBackendsSecondPass); |
|
|
|
|
num_started += kNumBackendsSecondPass; |
|
|
|
|
// Check client stats.
|
|
|
|
|
client_stats = balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
EXPECT_EQ(num_started, client_stats->total_successful_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats->total_requests_in_progress()); |
|
|
|
|
EXPECT_EQ(0U, client_stats->total_error_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats->total_dropped_requests()); |
|
|
|
|
load_report = balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
ASSERT_EQ(load_report.size(), 1UL); |
|
|
|
|
client_stats = std::move(load_report.front()); |
|
|
|
|
EXPECT_EQ(num_started, client_stats.total_successful_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats.total_requests_in_progress()); |
|
|
|
|
EXPECT_EQ(0U, client_stats.total_error_requests()); |
|
|
|
|
EXPECT_EQ(0U, client_stats.total_dropped_requests()); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
class ClientLoadReportingWithDropTest : public XdsEnd2endTest { |
|
|
|
|
@ -3352,15 +3544,18 @@ TEST_P(ClientLoadReportingWithDropTest, Vanilla) { |
|
|
|
|
::testing::Ge(KDropRateForLbAndThrottle * (1 - kErrorTolerance)), |
|
|
|
|
::testing::Le(KDropRateForLbAndThrottle * (1 + kErrorTolerance)))); |
|
|
|
|
// Check client stats.
|
|
|
|
|
ClientStats* client_stats = balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
EXPECT_EQ(num_drops, client_stats->total_dropped_requests()); |
|
|
|
|
std::vector<ClientStats> load_report = |
|
|
|
|
balancers_[0]->lrs_service()->WaitForLoadReport(); |
|
|
|
|
ASSERT_EQ(load_report.size(), 1UL); |
|
|
|
|
ClientStats& client_stats = load_report.front(); |
|
|
|
|
EXPECT_EQ(num_drops, client_stats.total_dropped_requests()); |
|
|
|
|
const size_t total_rpc = num_warmup + kNumRpcs; |
|
|
|
|
EXPECT_THAT( |
|
|
|
|
client_stats->dropped_requests(kLbDropType), |
|
|
|
|
client_stats.dropped_requests(kLbDropType), |
|
|
|
|
::testing::AllOf( |
|
|
|
|
::testing::Ge(total_rpc * kDropRateForLb * (1 - kErrorTolerance)), |
|
|
|
|
::testing::Le(total_rpc * kDropRateForLb * (1 + kErrorTolerance)))); |
|
|
|
|
EXPECT_THAT(client_stats->dropped_requests(kThrottleDropType), |
|
|
|
|
EXPECT_THAT(client_stats.dropped_requests(kThrottleDropType), |
|
|
|
|
::testing::AllOf( |
|
|
|
|
::testing::Ge(total_rpc * (1 - kDropRateForLb) * |
|
|
|
|
kDropRateForThrottle * (1 - kErrorTolerance)), |
|
|
|
|
@ -3417,6 +3612,11 @@ INSTANTIATE_TEST_SUITE_P(XdsTest, XdsResolverOnlyTest, |
|
|
|
|
TestType(true, true)), |
|
|
|
|
&TestTypeName); |
|
|
|
|
|
|
|
|
|
// XdsResolverLoadReprtingOnlyTest depends on XdsResolver and load reporting.
|
|
|
|
|
INSTANTIATE_TEST_SUITE_P(XdsTest, XdsResolverLoadReportingOnlyTest, |
|
|
|
|
::testing::Values(TestType(true, true)), |
|
|
|
|
&TestTypeName); |
|
|
|
|
|
|
|
|
|
INSTANTIATE_TEST_SUITE_P(XdsTest, LocalityMapTest, |
|
|
|
|
::testing::Values(TestType(false, true), |
|
|
|
|
TestType(false, false), |
|
|
|
|
|
Mark D. Roth
5 years ago