[XdsClient] Clean up mechanism used to store XdsServer in map keys.

For background, see #32826 and b/276944116.

PiperOrigin-RevId: 599876499
pull/35600/head
Mark D. Roth 10 months ago committed by Copybara-Service
parent 7e7b4fd1a4
commit b440474655
  1. 7
      src/core/ext/xds/xds_bootstrap.h
  2. 19
      src/core/ext/xds/xds_bootstrap_grpc.cc
  3. 3
      src/core/ext/xds/xds_bootstrap_grpc.h
  4. 81
      src/core/ext/xds/xds_client.cc
  5. 10
      src/core/ext/xds/xds_client.h
  6. 22
      src/core/ext/xds/xds_client_stats.cc
  7. 8
      src/core/ext/xds/xds_client_stats.h
  8. 28
      test/core/xds/xds_client_test.cc
  9. 6
      test/core/xds/xds_transport_fake.cc
  10. 2
      test/core/xds/xds_transport_fake.h

@ -50,6 +50,9 @@ class XdsBootstrap {
virtual bool Equals(const XdsServer& other) const = 0;
// Returns a key to be used for uniquely identifying this XdsServer.
virtual std::string Key() const = 0;
friend bool operator==(const XdsServer& a, const XdsServer& b) {
return a.Equals(b);
}
@ -80,10 +83,6 @@ class XdsBootstrap {
// Returns a pointer to the specified authority, or null if it does
// not exist in this bootstrap config.
virtual const Authority* LookupAuthority(const std::string& name) const = 0;
// If the server exists in the bootstrap config, returns a pointer to
// the XdsServer instance in the config. Otherwise, returns null.
virtual const XdsServer* FindXdsServer(const XdsServer& server) const = 0;
};
} // namespace grpc_core

@ -101,6 +101,10 @@ bool GrpcXdsBootstrap::GrpcXdsServer::Equals(const XdsServer& other) const {
server_features_ == o.server_features_);
}
std::string GrpcXdsBootstrap::GrpcXdsServer::Key() const {
return JsonDump(ToJson());
}
const JsonLoaderInterface* GrpcXdsBootstrap::GrpcXdsServer::JsonLoader(
const JsonArgs&) {
static const auto* loader =
@ -358,19 +362,4 @@ const XdsBootstrap::Authority* GrpcXdsBootstrap::LookupAuthority(
return nullptr;
}
const XdsBootstrap::XdsServer* GrpcXdsBootstrap::FindXdsServer(
const XdsBootstrap::XdsServer& server) const {
if (static_cast<const GrpcXdsServer&>(server) == servers_[0]) {
return &servers_[0];
}
for (auto& p : authorities_) {
const auto* authority_server =
static_cast<const GrpcXdsServer*>(p.second.server());
if (authority_server != nullptr && *authority_server == server) {
return authority_server;
}
}
return nullptr;
}
} // namespace grpc_core

@ -84,6 +84,8 @@ class GrpcXdsBootstrap : public XdsBootstrap {
bool Equals(const XdsServer& other) const override;
std::string Key() const override;
RefCountedPtr<ChannelCredsConfig> channel_creds_config() const {
return channel_creds_config_;
}
@ -132,7 +134,6 @@ class GrpcXdsBootstrap : public XdsBootstrap {
return node_.has_value() ? &*node_ : nullptr;
}
const Authority* LookupAuthority(const std::string& name) const override;
const XdsServer* FindXdsServer(const XdsServer& server) const override;
const std::string& client_default_listener_resource_name_template() const {
return client_default_listener_resource_name_template_;

@ -488,7 +488,7 @@ void XdsClient::XdsChannel::Orphan() ABSL_NO_THREAD_SAFETY_ANALYSIS {
// At this time, all strong refs are removed, remove from channel map to
// prevent subsequent subscription from trying to use this XdsChannel as
// it is shutting down.
xds_client_->xds_server_channel_map_.erase(&server_);
xds_client_->xds_channel_map_.erase(server_.Key());
ads_call_.reset();
lrs_call_.reset();
}
@ -510,7 +510,7 @@ void XdsClient::XdsChannel::MaybeStartLrsCall() {
}
void XdsClient::XdsChannel::StopLrsCallLocked() {
xds_client_->xds_load_report_server_map_.erase(&server_);
xds_client_->xds_load_report_server_map_.erase(server_.Key());
lrs_call_.reset();
}
@ -1308,8 +1308,8 @@ void XdsClient::XdsChannel::LrsCall::Orphan() {
void XdsClient::XdsChannel::LrsCall::MaybeScheduleNextReportLocked() {
// If there are no more registered stats to report, cancel the call.
auto it =
xds_client()->xds_load_report_server_map_.find(&xds_channel()->server_);
auto it = xds_client()->xds_load_report_server_map_.find(
xds_channel()->server_.Key());
if (it == xds_client()->xds_load_report_server_map_.end() ||
it->second.load_report_map.empty()) {
it->second.xds_channel->StopLrsCallLocked();
@ -1531,14 +1531,15 @@ void XdsClient::Orphan() {
RefCountedPtr<XdsClient::XdsChannel> XdsClient::GetOrCreateXdsChannelLocked(
const XdsBootstrap::XdsServer& server, const char* reason) {
auto it = xds_server_channel_map_.find(&server);
if (it != xds_server_channel_map_.end()) {
std::string key = server.Key();
auto it = xds_channel_map_.find(key);
if (it != xds_channel_map_.end()) {
return it->second->Ref(DEBUG_LOCATION, reason);
}
// Channel not found, so create a new one.
auto xds_channel =
MakeRefCounted<XdsChannel>(WeakRef(DEBUG_LOCATION, "XdsChannel"), server);
xds_server_channel_map_[&server] = xds_channel.get();
xds_channel_map_[std::move(key)] = xds_channel.get();
return xds_channel;
}
@ -1580,14 +1581,6 @@ void XdsClient::WatchResource(const XdsResourceType* type,
xds_server = authority->server();
}
if (xds_server == nullptr) xds_server = &bootstrap_->server();
// Canonify the xDS server instance, so that we make sure we're using
// the same instance as will be used in AddClusterDropStats() and
// AddClusterLocalityStats(). This may yield a different result than
// the logic above if the same server is listed both in the authority
// and as the top-level server.
// TODO(roth): This is really ugly -- need to find a better way to
// index the xDS server than by address here.
xds_server = bootstrap_->FindXdsServer(*xds_server);
{
MutexLock lock(&mu_);
MaybeRegisterResourceTypeLocked(type);
@ -1781,23 +1774,21 @@ std::string XdsClient::ConstructFullXdsResourceName(
RefCountedPtr<XdsClusterDropStats> XdsClient::AddClusterDropStats(
const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name,
absl::string_view eds_service_name) {
const auto* server = bootstrap_->FindXdsServer(xds_server);
if (server == nullptr) return nullptr;
auto key =
std::make_pair(std::string(cluster_name), std::string(eds_service_name));
RefCountedPtr<XdsClusterDropStats> cluster_drop_stats;
{
MutexLock lock(&mu_);
// We jump through some hoops here to make sure that the const
// XdsBootstrap::XdsServer& and absl::string_views
// stored in the XdsClusterDropStats object point to the
// XdsBootstrap::XdsServer and strings
// in the load_report_map_ key, so that they have the same lifetime.
auto server_it =
xds_load_report_server_map_.emplace(server, LoadReportServer()).first;
// We jump through some hoops here to make sure that the
// absl::string_views stored in the XdsClusterDropStats object point
// to the strings in the xds_load_report_server_map_ keys, so that
// they have the same lifetime.
auto server_it = xds_load_report_server_map_
.emplace(xds_server.Key(), LoadReportServer())
.first;
if (server_it->second.xds_channel == nullptr) {
server_it->second.xds_channel =
GetOrCreateXdsChannelLocked(*server, "load report map (drop stats)");
server_it->second.xds_channel = GetOrCreateXdsChannelLocked(
xds_server, "load report map (drop stats)");
}
auto load_report_it = server_it->second.load_report_map
.emplace(std::move(key), LoadReportState())
@ -1812,7 +1803,7 @@ RefCountedPtr<XdsClusterDropStats> XdsClient::AddClusterDropStats(
load_report_state.drop_stats->GetSnapshotAndReset();
}
cluster_drop_stats = MakeRefCounted<XdsClusterDropStats>(
Ref(DEBUG_LOCATION, "DropStats"), *server,
Ref(DEBUG_LOCATION, "DropStats"), server_it->first /*xds_server*/,
load_report_it->first.first /*cluster_name*/,
load_report_it->first.second /*eds_service_name*/);
load_report_state.drop_stats = cluster_drop_stats.get();
@ -1824,13 +1815,11 @@ RefCountedPtr<XdsClusterDropStats> XdsClient::AddClusterDropStats(
}
void XdsClient::RemoveClusterDropStats(
const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name,
absl::string_view xds_server_key, absl::string_view cluster_name,
absl::string_view eds_service_name,
XdsClusterDropStats* cluster_drop_stats) {
const auto* server = bootstrap_->FindXdsServer(xds_server);
if (server == nullptr) return;
MutexLock lock(&mu_);
auto server_it = xds_load_report_server_map_.find(server);
auto server_it = xds_load_report_server_map_.find(xds_server_key);
if (server_it == xds_load_report_server_map_.end()) return;
auto load_report_it = server_it->second.load_report_map.find(
std::make_pair(std::string(cluster_name), std::string(eds_service_name)));
@ -1849,23 +1838,21 @@ RefCountedPtr<XdsClusterLocalityStats> XdsClient::AddClusterLocalityStats(
const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name,
absl::string_view eds_service_name,
RefCountedPtr<XdsLocalityName> locality) {
const auto* server = bootstrap_->FindXdsServer(xds_server);
if (server == nullptr) return nullptr;
auto key =
std::make_pair(std::string(cluster_name), std::string(eds_service_name));
RefCountedPtr<XdsClusterLocalityStats> cluster_locality_stats;
{
MutexLock lock(&mu_);
// We jump through some hoops here to make sure that the const
// XdsBootstrap::XdsServer& and absl::string_views
// stored in the XdsClusterDropStats object point to the
// XdsBootstrap::XdsServer and strings
// in the load_report_map_ key, so that they have the same lifetime.
auto server_it =
xds_load_report_server_map_.emplace(server, LoadReportServer()).first;
// We jump through some hoops here to make sure that the
// absl::string_views stored in the XdsClusterDropStats object point
// to the strings in the xds_load_report_server_map_ keys, so that
// they have the same lifetime.
auto server_it = xds_load_report_server_map_
.emplace(xds_server.Key(), LoadReportServer())
.first;
if (server_it->second.xds_channel == nullptr) {
server_it->second.xds_channel = GetOrCreateXdsChannelLocked(
*server, "load report map (locality stats)");
xds_server, "load report map (locality stats)");
}
auto load_report_it = server_it->second.load_report_map
.emplace(std::move(key), LoadReportState())
@ -1882,7 +1869,7 @@ RefCountedPtr<XdsClusterLocalityStats> XdsClient::AddClusterLocalityStats(
locality_state.locality_stats->GetSnapshotAndReset();
}
cluster_locality_stats = MakeRefCounted<XdsClusterLocalityStats>(
Ref(DEBUG_LOCATION, "LocalityStats"), *server,
Ref(DEBUG_LOCATION, "LocalityStats"), server_it->first /*xds_server*/,
load_report_it->first.first /*cluster_name*/,
load_report_it->first.second /*eds_service_name*/,
std::move(locality));
@ -1895,14 +1882,12 @@ RefCountedPtr<XdsClusterLocalityStats> XdsClient::AddClusterLocalityStats(
}
void XdsClient::RemoveClusterLocalityStats(
const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name,
absl::string_view xds_server_key, absl::string_view cluster_name,
absl::string_view eds_service_name,
const RefCountedPtr<XdsLocalityName>& locality,
XdsClusterLocalityStats* cluster_locality_stats) {
const auto* server = bootstrap_->FindXdsServer(xds_server);
if (server == nullptr) return;
MutexLock lock(&mu_);
auto server_it = xds_load_report_server_map_.find(server);
auto server_it = xds_load_report_server_map_.find(xds_server_key);
if (server_it == xds_load_report_server_map_.end()) return;
auto load_report_it = server_it->second.load_report_map.find(
std::make_pair(std::string(cluster_name), std::string(eds_service_name)));
@ -1922,7 +1907,7 @@ void XdsClient::RemoveClusterLocalityStats(
void XdsClient::ResetBackoff() {
MutexLock lock(&mu_);
for (auto& p : xds_server_channel_map_) {
for (auto& p : xds_channel_map_) {
p.second->ResetBackoff();
}
}
@ -1969,7 +1954,7 @@ XdsApi::ClusterLoadReportMap XdsClient::BuildLoadReportSnapshotLocked(
gpr_log(GPR_INFO, "[xds_client %p] start building load report", this);
}
XdsApi::ClusterLoadReportMap snapshot_map;
auto server_it = xds_load_report_server_map_.find(&xds_server);
auto server_it = xds_load_report_server_map_.find(xds_server.Key());
if (server_it == xds_load_report_server_map_.end()) return snapshot_map;
auto& load_report_map = server_it->second.load_report_map;
for (auto load_report_it = load_report_map.begin();

@ -127,7 +127,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
RefCountedPtr<XdsClusterDropStats> AddClusterDropStats(
const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name,
absl::string_view eds_service_name);
void RemoveClusterDropStats(const XdsBootstrap::XdsServer& xds_server,
void RemoveClusterDropStats(absl::string_view xds_server,
absl::string_view cluster_name,
absl::string_view eds_service_name,
XdsClusterDropStats* cluster_drop_stats);
@ -139,7 +139,7 @@ class XdsClient : public DualRefCounted<XdsClient> {
absl::string_view eds_service_name,
RefCountedPtr<XdsLocalityName> locality);
void RemoveClusterLocalityStats(
const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name,
absl::string_view xds_server, absl::string_view cluster_name,
absl::string_view eds_service_name,
const RefCountedPtr<XdsLocalityName>& locality,
XdsClusterLocalityStats* cluster_locality_stats);
@ -329,15 +329,13 @@ class XdsClient : public DualRefCounted<XdsClient> {
upb::DefPool def_pool_ ABSL_GUARDED_BY(mu_);
// Map of existing xDS server channels.
// Key is owned by the bootstrap config.
std::map<const XdsBootstrap::XdsServer*, XdsChannel*> xds_server_channel_map_
std::map<std::string /*XdsServer key*/, XdsChannel*> xds_channel_map_
ABSL_GUARDED_BY(mu_);
std::map<std::string /*authority*/, AuthorityState> authority_state_map_
ABSL_GUARDED_BY(mu_);
// Key is owned by the bootstrap config.
std::map<const XdsBootstrap::XdsServer*, LoadReportServer>
std::map<std::string /*XdsServer key*/, LoadReportServer, std::less<>>
xds_load_report_server_map_ ABSL_GUARDED_BY(mu_);
// Stores started watchers whose resource name was not parsed successfully,

@ -40,10 +40,10 @@ uint64_t GetAndResetCounter(std::atomic<uint64_t>* from) {
// XdsClusterDropStats
//
XdsClusterDropStats::XdsClusterDropStats(
RefCountedPtr<XdsClient> xds_client,
const XdsBootstrap::XdsServer& lrs_server, absl::string_view cluster_name,
absl::string_view eds_service_name)
XdsClusterDropStats::XdsClusterDropStats(RefCountedPtr<XdsClient> xds_client,
absl::string_view lrs_server,
absl::string_view cluster_name,
absl::string_view eds_service_name)
: RefCounted(GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_refcount_trace)
? "XdsClusterDropStats"
: nullptr),
@ -53,7 +53,7 @@ XdsClusterDropStats::XdsClusterDropStats(
eds_service_name_(eds_service_name) {
if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
gpr_log(GPR_INFO, "[xds_client %p] created drop stats %p for {%s, %s, %s}",
xds_client_.get(), this, lrs_server_.server_uri().c_str(),
xds_client_.get(), this, std::string(lrs_server_).c_str(),
std::string(cluster_name_).c_str(),
std::string(eds_service_name_).c_str());
}
@ -63,7 +63,7 @@ XdsClusterDropStats::~XdsClusterDropStats() {
if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
gpr_log(GPR_INFO,
"[xds_client %p] destroying drop stats %p for {%s, %s, %s}",
xds_client_.get(), this, lrs_server_.server_uri().c_str(),
xds_client_.get(), this, std::string(lrs_server_).c_str(),
std::string(cluster_name_).c_str(),
std::string(eds_service_name_).c_str());
}
@ -94,9 +94,9 @@ void XdsClusterDropStats::AddCallDropped(const std::string& category) {
//
XdsClusterLocalityStats::XdsClusterLocalityStats(
RefCountedPtr<XdsClient> xds_client,
const XdsBootstrap::XdsServer& lrs_server, absl::string_view cluster_name,
absl::string_view eds_service_name, RefCountedPtr<XdsLocalityName> name)
RefCountedPtr<XdsClient> xds_client, absl::string_view lrs_server,
absl::string_view cluster_name, absl::string_view eds_service_name,
RefCountedPtr<XdsLocalityName> name)
: RefCounted(GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_refcount_trace)
? "XdsClusterLocalityStats"
: nullptr),
@ -108,7 +108,7 @@ XdsClusterLocalityStats::XdsClusterLocalityStats(
if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
gpr_log(GPR_INFO,
"[xds_client %p] created locality stats %p for {%s, %s, %s, %s}",
xds_client_.get(), this, lrs_server_.server_uri().c_str(),
xds_client_.get(), this, std::string(lrs_server_).c_str(),
std::string(cluster_name_).c_str(),
std::string(eds_service_name_).c_str(),
name_->AsHumanReadableString().c_str());
@ -119,7 +119,7 @@ XdsClusterLocalityStats::~XdsClusterLocalityStats() {
if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) {
gpr_log(GPR_INFO,
"[xds_client %p] destroying locality stats %p for {%s, %s, %s, %s}",
xds_client_.get(), this, lrs_server_.server_uri().c_str(),
xds_client_.get(), this, std::string(lrs_server_).c_str(),
std::string(cluster_name_).c_str(),
std::string(eds_service_name_).c_str(),
name_->AsHumanReadableString().c_str());

@ -141,7 +141,7 @@ class XdsClusterDropStats : public RefCounted<XdsClusterDropStats> {
};
XdsClusterDropStats(RefCountedPtr<XdsClient> xds_client,
const XdsBootstrap::XdsServer& lrs_server,
absl::string_view lrs_server,
absl::string_view cluster_name,
absl::string_view eds_service_name);
~XdsClusterDropStats() override;
@ -154,7 +154,7 @@ class XdsClusterDropStats : public RefCounted<XdsClusterDropStats> {
private:
RefCountedPtr<XdsClient> xds_client_;
const XdsBootstrap::XdsServer& lrs_server_;
absl::string_view lrs_server_;
absl::string_view cluster_name_;
absl::string_view eds_service_name_;
std::atomic<uint64_t> uncategorized_drops_{0};
@ -215,7 +215,7 @@ class XdsClusterLocalityStats : public RefCounted<XdsClusterLocalityStats> {
};
XdsClusterLocalityStats(RefCountedPtr<XdsClient> xds_client,
const XdsBootstrap::XdsServer& lrs_server_,
absl::string_view lrs_server,
absl::string_view cluster_name,
absl::string_view eds_service_name,
RefCountedPtr<XdsLocalityName> name);
@ -244,7 +244,7 @@ class XdsClusterLocalityStats : public RefCounted<XdsClusterLocalityStats> {
};
RefCountedPtr<XdsClient> xds_client_;
const XdsBootstrap::XdsServer& lrs_server_;
absl::string_view lrs_server_;
absl::string_view cluster_name_;
absl::string_view eds_service_name_;
RefCountedPtr<XdsLocalityName> name_;

@ -131,6 +131,9 @@ class XdsClientTest : public ::testing::Test {
return server_uri_ == o.server_uri_ &&
ignore_resource_deletion_ == o.ignore_resource_deletion_;
}
std::string Key() const override {
return absl::StrCat(server_uri_, "#", ignore_resource_deletion_);
}
void set_server_uri(std::string server_uri) {
server_uri_ = std::move(server_uri);
@ -200,18 +203,6 @@ class XdsClientTest : public ::testing::Test {
if (it == authorities_.end()) return nullptr;
return &it->second;
}
const XdsServer* FindXdsServer(const XdsServer& server) const override {
const auto& fake_server = static_cast<const FakeXdsServer&>(server);
if (fake_server == server_) return &server_;
for (const auto& p : authorities_) {
const auto* authority_server =
static_cast<const FakeXdsServer*>(p.second.server());
if (authority_server != nullptr && *authority_server == fake_server) {
return authority_server;
}
}
return nullptr;
}
private:
FakeXdsServer server_;
@ -654,21 +645,16 @@ class XdsClientTest : public ::testing::Test {
}
RefCountedPtr<FakeXdsTransportFactory::FakeStreamingCall> WaitForAdsStream(
const XdsBootstrap::XdsServer& server,
const XdsBootstrap::XdsServer& xds_server,
absl::Duration timeout = absl::Seconds(5)) {
const auto* xds_server = xds_client_->bootstrap().FindXdsServer(server);
GPR_ASSERT(xds_server != nullptr);
return transport_factory_->WaitForStream(
*xds_server, FakeXdsTransportFactory::kAdsMethod,
xds_server, FakeXdsTransportFactory::kAdsMethod,
timeout * grpc_test_slowdown_factor());
}
void TriggerConnectionFailure(const XdsBootstrap::XdsServer& server,
void TriggerConnectionFailure(const XdsBootstrap::XdsServer& xds_server,
absl::Status status) {
const auto* xds_server = xds_client_->bootstrap().FindXdsServer(server);
GPR_ASSERT(xds_server != nullptr);
transport_factory_->TriggerConnectionFailure(*xds_server,
std::move(status));
transport_factory_->TriggerConnectionFailure(xds_server, std::move(status));
}
RefCountedPtr<FakeXdsTransportFactory::FakeStreamingCall> WaitForAdsStream(

@ -209,7 +209,7 @@ void FakeXdsTransportFactory::FakeXdsTransport::TriggerConnectionFailure(
void FakeXdsTransportFactory::FakeXdsTransport::Orphan() {
{
MutexLock lock(&factory_->mu_);
auto it = factory_->transport_map_.find(&server_);
auto it = factory_->transport_map_.find(server_.Key());
if (it != factory_->transport_map_.end() && it->second == this) {
factory_->transport_map_.erase(it);
}
@ -277,7 +277,7 @@ FakeXdsTransportFactory::Create(
std::function<void(absl::Status)> on_connectivity_failure,
absl::Status* /*status*/) {
MutexLock lock(&mu_);
auto& entry = transport_map_[&server];
auto& entry = transport_map_[server.Key()];
GPR_ASSERT(entry == nullptr);
auto transport = MakeOrphanable<FakeXdsTransport>(
RefAsSubclass<FakeXdsTransportFactory>(), server,
@ -316,7 +316,7 @@ FakeXdsTransportFactory::WaitForStream(const XdsBootstrap::XdsServer& server,
RefCountedPtr<FakeXdsTransportFactory::FakeXdsTransport>
FakeXdsTransportFactory::GetTransport(const XdsBootstrap::XdsServer& server) {
MutexLock lock(&mu_);
return transport_map_[&server];
return transport_map_[server.Key()];
}
} // namespace grpc_core

@ -251,7 +251,7 @@ class FakeXdsTransportFactory : public XdsTransportFactory {
const XdsBootstrap::XdsServer& server);
Mutex mu_;
std::map<const XdsBootstrap::XdsServer*, RefCountedPtr<FakeXdsTransport>>
std::map<std::string /*XdsServer key*/, RefCountedPtr<FakeXdsTransport>>
transport_map_ ABSL_GUARDED_BY(&mu_);
bool auto_complete_messages_from_client_ ABSL_GUARDED_BY(&mu_) = true;
bool abort_on_undrained_messages_ ABSL_GUARDED_BY(&mu_) = true;

Loading…
Cancel
Save