@ -22,7 +22,6 @@
# include <stdlib.h>
# include <algorithm>
# include <atomic>
# include <cmath>
# include <memory>
# include <string>
@ -30,7 +29,6 @@
# include <vector>
# include "absl/base/attributes.h"
# include "absl/base/thread_annotations.h"
# include "absl/container/inlined_vector.h"
# include "absl/status/status.h"
# include "absl/status/statusor.h"
@ -39,11 +37,10 @@
# include "absl/strings/string_view.h"
# include "absl/types/optional.h"
# include <grpc/grpc.h>
# define XXH_INLINE_ALL
# include "xxhash.h"
# include <grpc/grpc.h>
# include <grpc/impl/connectivity_state.h>
# include <grpc/support/log.h>
@ -55,8 +52,8 @@
# include "src/core/lib/debug/trace.h"
# include "src/core/lib/gprpp/debug_location.h"
# include "src/core/lib/gprpp/orphanable.h"
# include "src/core/lib/gprpp/ref_counted.h"
# include "src/core/lib/gprpp/ref_counted_ptr.h"
# include "src/core/lib/gprpp/sync.h"
# include "src/core/lib/gprpp/unique_type_name.h"
# include "src/core/lib/gprpp/work_serializer.h"
# include "src/core/lib/iomgr/closure.h"
@ -143,8 +140,6 @@ class RingHash : public LoadBalancingPolicy {
void ResetBackoffLocked ( ) override ;
private :
~ RingHash ( ) override ;
// Forward declaration.
class RingHashSubchannelList ;
@ -165,13 +160,11 @@ class RingHash : public LoadBalancingPolicy {
const ServerAddress & address ( ) const { return address_ ; }
grpc_connectivity_state GetConnectivityS tate( ) const {
return connectivity_state_ . load ( std : : memory_order_relaxed ) ;
grpc_connectivity_state logical_connectivity_s tate( ) const {
return logical_ connectivity_state_;
}
absl : : Status GetConnectivityStatus ( ) const {
MutexLock lock ( & mu_ ) ;
return connectivity_status_ ;
const absl : : Status & logical_connectivity_status ( ) const {
return logical_connectivity_status_ ;
}
private :
@ -188,20 +181,28 @@ class RingHash : public LoadBalancingPolicy {
// subchannel in some cases; for example, once this is set to
// TRANSIENT_FAILURE, we do not change it again until we get READY,
// so we skip any interim stops in CONNECTING.
// Uses an atomic so that it can be accessed outside of the WorkSerializer.
std : : atomic < grpc_connectivity_state > connectivity_state_ { GRPC_CHANNEL_IDLE } ;
mutable Mutex mu_ ;
absl : : Status connectivity_status_ ABSL_GUARDED_BY ( & mu_ ) ;
grpc_connectivity_state logical_connectivity_state_ = GRPC_CHANNEL_IDLE ;
absl : : Status logical_connectivity_status_ ;
} ;
// A list of subchannels and the ring containing those subchannels.
class RingHashSubchannelList
: public SubchannelList < RingHashSubchannelList , RingHashSubchannelData > {
public :
struct RingEntry {
uint64_t hash ;
RingHashSubchannelData * subchannel ;
class Ring : public RefCounted < Ring > {
public :
struct RingEntry {
uint64_t hash ;
size_t subchannel_index ;
} ;
Ring ( RingHashLbConfig * config , RingHashSubchannelList * subchannel_list ,
const ChannelArgs & args ) ;
const std : : vector < RingEntry > & ring ( ) const { return ring_ ; }
private :
std : : vector < RingEntry > ring_ ;
} ;
RingHashSubchannelList ( RingHash * policy , ServerAddressList addresses ,
@ -212,7 +213,7 @@ class RingHash : public LoadBalancingPolicy {
p - > Unref ( DEBUG_LOCATION , " subchannel_list " ) ;
}
const std : : vecto r< RingEntry > & ring ( ) const { return ring_ ; }
RefCountedPt r< Ring > ring ( ) { return ring_ ; }
// Updates the counters of subchannels in each state when a
// subchannel transitions from old_state to new_state.
@ -236,7 +237,7 @@ class RingHash : public LoadBalancingPolicy {
size_t num_connecting_ = 0 ;
size_t num_transient_failure_ = 0 ;
std : : vecto r< RingEntry > ring_ ;
RefCountedPt r< Ring > ring_ ;
// The index of the subchannel currently doing an internally
// triggered connection attempt, if any.
@ -251,24 +252,31 @@ class RingHash : public LoadBalancingPolicy {
class Picker : public SubchannelPicker {
public :
explicit Picker ( RefCountedPtr < RingHashSubchannelList > subchannel_list )
: subchannel_list_ ( std : : move ( subchannel_list ) ) { }
~ Picker ( ) override {
// Hop into WorkSerializer to unref the subchannel list, since that may
// trigger the unreffing of the underlying subchannels.
MakeOrphanable < WorkSerializerRunner > ( std : : move ( subchannel_list_ ) ) ;
Picker ( RefCountedPtr < RingHash > ring_hash_lb ,
RingHashSubchannelList * subchannel_list )
: ring_hash_lb_ ( std : : move ( ring_hash_lb ) ) ,
ring_ ( subchannel_list - > ring ( ) ) {
subchannels_ . reserve ( subchannel_list - > num_subchannels ( ) ) ;
for ( size_t i = 0 ; i < subchannel_list - > num_subchannels ( ) ; + + i ) {
RingHashSubchannelData * subchannel_data =
subchannel_list - > subchannel ( i ) ;
subchannels_ . emplace_back (
SubchannelInfo { subchannel_data - > subchannel ( ) - > Ref ( ) ,
subchannel_data - > logical_connectivity_state ( ) ,
subchannel_data - > logical_connectivity_status ( ) } ) ;
}
}
PickResult Pick ( PickArgs args ) override ;
private :
// An interface for running a callback in the control plane WorkSerializer.
class WorkSerializerRunner : public Orphanable {
// A fire-and-forget class that schedules subchannel connection attempts
// on the control plane WorkSerializer.
class SubchannelConnectionAttempter : public Orphanable {
public :
explicit WorkSerializerRunn er(
RefCountedPtr < RingHashSubchannelList > subchannel_list )
: subchannel_list _( std : : move ( subchannel_list ) ) {
explicit SubchannelConnectionAttempt er(
RefCountedPtr < RingHash > ring_hash_lb )
: ring_hash_lb _( std : : move ( ring_hash_lb ) ) {
GRPC_CLOSURE_INIT ( & closure_ , RunInExecCtx , this , nullptr ) ;
}
@ -278,56 +286,43 @@ class RingHash : public LoadBalancingPolicy {
ExecCtx : : Run ( DEBUG_LOCATION , & closure_ , absl : : OkStatus ( ) ) ;
}
// Will be invoked inside of the WorkSerializer.
virtual void Run ( ) { }
protected :
RingHash * ring_hash_lb ( ) const {
return static_cast < RingHash * > ( subchannel_list_ - > policy ( ) ) ;
void AddSubchannel ( RefCountedPtr < SubchannelInterface > subchannel ) {
subchannels_ . push_back ( std : : move ( subchannel ) ) ;
}
private :
static void RunInExecCtx ( void * arg , grpc_error_handle /*error*/ ) {
auto * self = static_cast < WorkSerializerRunn er* > ( arg ) ;
self - > ring_hash_lb ( ) - > work_serializer ( ) - > Run (
auto * self = static_cast < SubchannelConnectionAttempt er* > ( arg ) ;
self - > ring_hash_lb_ - > work_serializer ( ) - > Run (
[ self ] ( ) {
self - > Run ( ) ;
if ( ! self - > ring_hash_lb_ - > shutdown_ ) {
for ( auto & subchannel : self - > subchannels_ ) {
subchannel - > RequestConnection ( ) ;
}
}
delete self ;
} ,
DEBUG_LOCATION ) ;
}
RefCountedPtr < RingHashSubchannelList > subchannel_list _;
RefCountedPtr < RingHash > ring_hash_lb _;
grpc_closure closure_ ;
std : : vector < RefCountedPtr < SubchannelInterface > > subchannels_ ;
} ;
// A fire-and-forget class that schedules subchannel connection attempts
// on the control plane WorkSerializer.
class SubchannelConnectionAttempter : public WorkSerializerRunner {
public :
explicit SubchannelConnectionAttempter (
RefCountedPtr < RingHashSubchannelList > subchannel_list )
: WorkSerializerRunner ( std : : move ( subchannel_list ) ) { }
void AddSubchannel ( RefCountedPtr < SubchannelInterface > subchannel ) {
subchannels_ . push_back ( std : : move ( subchannel ) ) ;
}
void Run ( ) override {
if ( ! ring_hash_lb ( ) - > shutdown_ ) {
for ( auto & subchannel : subchannels_ ) {
subchannel - > RequestConnection ( ) ;
}
}
}
private :
std : : vector < RefCountedPtr < SubchannelInterface > > subchannels_ ;
struct SubchannelInfo {
RefCountedPtr < SubchannelInterface > subchannel ;
grpc_connectivity_state state ;
absl : : Status status ;
} ;
RefCountedPtr < RingHashSubchannelList > subchannel_list_ ;
RefCountedPtr < RingHash > ring_hash_lb_ ;
RefCountedPtr < RingHashSubchannelList : : Ring > ring_ ;
std : : vector < SubchannelInfo > subchannels_ ;
} ;
~ RingHash ( ) override ;
void ShutdownLocked ( ) override ;
// Current config from resolver.
@ -353,7 +348,7 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
return PickResult : : Fail (
absl : : InternalError ( " ring hash value is not a number " ) ) ;
}
const auto & ring = subchannel_list _- > ring ( ) ;
const auto & ring = ring _- > ring ( ) ;
// Ported from https://github.com/RJ/ketama/blob/master/libketama/ketama.c
// (ketama_get_server) NOTE: The algorithm depends on using signed integers
// for lowp, highp, and first_index. Do not change them!
@ -386,27 +381,25 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
[ & ] ( RefCountedPtr < SubchannelInterface > subchannel ) {
if ( subchannel_connection_attempter = = nullptr ) {
subchannel_connection_attempter =
MakeOrphanable < SubchannelConnectionAttempter > (
subchannel_list_ - > Ref ( DEBUG_LOCATION ,
" SubchannelConnectionAttempter " ) ) ;
MakeOrphanable < SubchannelConnectionAttempter > ( ring_hash_lb_ - > Ref (
DEBUG_LOCATION , " SubchannelConnectionAttempter " ) ) ;
}
subchannel_connection_attempter - > AddSubchannel ( std : : move ( subchannel ) ) ;
} ;
switch ( ring [ first_index ] . subchannel - > GetConnectivityState ( ) ) {
SubchannelInfo & first_subchannel =
subchannels_ [ ring [ first_index ] . subchannel_index ] ;
switch ( first_subchannel . state ) {
case GRPC_CHANNEL_READY :
return PickResult : : Complete (
ring [ first_index ] . subchannel - > subchannel ( ) - > Ref ( ) ) ;
return PickResult : : Complete ( first_subchannel . subchannel ) ;
case GRPC_CHANNEL_IDLE :
ScheduleSubchannelConnectionAttempt (
ring [ first_index ] . subchannel - > subchannel ( ) - > Ref ( ) ) ;
ScheduleSubchannelConnectionAttempt ( first_subchannel . subchannel ) ;
ABSL_FALLTHROUGH_INTENDED ;
case GRPC_CHANNEL_CONNECTING :
return PickResult : : Queue ( ) ;
default : // GRPC_CHANNEL_TRANSIENT_FAILURE
break ;
}
ScheduleSubchannelConnectionAttempt (
ring [ first_index ] . subchannel - > subchannel ( ) - > Ref ( ) ) ;
ScheduleSubchannelConnectionAttempt ( first_subchannel . subchannel ) ;
// Loop through remaining subchannels to find one in READY.
// On the way, we make sure the right set of connection attempts
// will happen.
@ -414,19 +407,17 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
bool found_first_non_failed = false ;
for ( size_t i = 1 ; i < ring . size ( ) ; + + i ) {
const auto & entry = ring [ ( first_index + i ) % ring . size ( ) ] ;
if ( entry . subchannel = = ring [ first_index ] . subchannel ) {
if ( entry . subchannel_index = = ring [ first_index ] . subchannel_index ) {
continue ;
}
grpc_connectivity_state connectivity_state =
entry . subchannel - > GetConnectivityState ( ) ;
if ( connectivity_state = = GRPC_CHANNEL_READY ) {
return PickResult : : Complete ( entry . subchannel - > subchannel ( ) - > Ref ( ) ) ;
SubchannelInfo & subchannel_info = subchannels_ [ entry . subchannel_index ] ;
if ( subchannel_info . state = = GRPC_CHANNEL_READY ) {
return PickResult : : Complete ( subchannel_info . subchannel ) ;
}
if ( ! found_second_subchannel ) {
switch ( connectivity_ state) {
switch ( subchannel_info . state ) {
case GRPC_CHANNEL_IDLE :
ScheduleSubchannelConnectionAttempt (
entry . subchannel - > subchannel ( ) - > Ref ( ) ) ;
ScheduleSubchannelConnectionAttempt ( subchannel_info . subchannel ) ;
ABSL_FALLTHROUGH_INTENDED ;
case GRPC_CHANNEL_CONNECTING :
return PickResult : : Queue ( ) ;
@ -436,13 +427,11 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
found_second_subchannel = true ;
}
if ( ! found_first_non_failed ) {
if ( connectivity_state = = GRPC_CHANNEL_TRANSIENT_FAILURE ) {
ScheduleSubchannelConnectionAttempt (
entry . subchannel - > subchannel ( ) - > Ref ( ) ) ;
if ( subchannel_info . state = = GRPC_CHANNEL_TRANSIENT_FAILURE ) {
ScheduleSubchannelConnectionAttempt ( subchannel_info . subchannel ) ;
} else {
if ( connectivity_state = = GRPC_CHANNEL_IDLE ) {
ScheduleSubchannelConnectionAttempt (
entry . subchannel - > subchannel ( ) - > Ref ( ) ) ;
if ( subchannel_info . state = = GRPC_CHANNEL_IDLE ) {
ScheduleSubchannelConnectionAttempt ( subchannel_info . subchannel ) ;
}
found_first_non_failed = true ;
}
@ -450,27 +439,16 @@ RingHash::PickResult RingHash::Picker::Pick(PickArgs args) {
}
return PickResult : : Fail ( absl : : UnavailableError ( absl : : StrCat (
" ring hash cannot find a connected subchannel; first failure: " ,
ring [ first_index ] . subchannel - > GetConnectivityStatus ( ) . ToString ( ) ) ) ) ;
first_subchannel . status . ToString ( ) ) ) ) ;
}
//
// RingHash::RingHashSubchannelList
// RingHash::RingHashSubchannelList::Ring
//
RingHash : : RingHashSubchannelList : : RingHashSubchannelList (
RingHash * policy , ServerAddressList addresses , const ChannelArgs & args )
: SubchannelList ( policy ,
( GRPC_TRACE_FLAG_ENABLED ( grpc_lb_ring_hash_trace )
? " RingHashSubchannelList "
: nullptr ) ,
std : : move ( addresses ) , policy - > channel_control_helper ( ) ,
args ) ,
num_idle_ ( num_subchannels ( ) ) {
// Need to maintain a ref to the LB policy as long as we maintain
// any references to subchannels, since the subchannels'
// pollset_sets will include the LB policy's pollset_set.
policy - > Ref ( DEBUG_LOCATION , " subchannel_list " ) . release ( ) ;
// Construct the ring.
RingHash : : RingHashSubchannelList : : Ring : : Ring (
RingHashLbConfig * config , RingHashSubchannelList * subchannel_list ,
const ChannelArgs & args ) {
// Store the weights while finding the sum.
struct AddressWeight {
std : : string address ;
@ -481,9 +459,9 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
} ;
std : : vector < AddressWeight > address_weights ;
size_t sum = 0 ;
address_weights . reserve ( num_subchannels ( ) ) ;
for ( size_t i = 0 ; i < num_subchannels ( ) ; + + i ) {
RingHashSubchannelData * sd = subchannel ( i ) ;
address_weights . reserve ( subchannel_list - > num_subchannels ( ) ) ;
for ( size_t i = 0 ; i < subchannel_list - > num_subchannels ( ) ; + + i ) {
RingHashSubchannelData * sd = subchannel_list - > subchannel ( i ) ;
const ServerAddressWeightAttribute * weight_attribute = static_cast <
const ServerAddressWeightAttribute * > ( sd - > address ( ) . GetAttribute (
ServerAddressWeightAttribute : : kServerAddressWeightAttributeKey ) ) ;
@ -517,10 +495,8 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
// to fit.
const size_t ring_size_cap = args . GetInt ( GRPC_ARG_RING_HASH_LB_RING_SIZE_CAP )
. value_or ( kRingSizeCapDefault ) ;
const size_t min_ring_size =
std : : min ( policy - > config_ - > min_ring_size ( ) , ring_size_cap ) ;
const size_t max_ring_size =
std : : min ( policy - > config_ - > max_ring_size ( ) , ring_size_cap ) ;
const size_t min_ring_size = std : : min ( config - > min_ring_size ( ) , ring_size_cap ) ;
const size_t max_ring_size = std : : min ( config - > max_ring_size ( ) , ring_size_cap ) ;
const double scale = std : : min (
std : : ceil ( min_normalized_weight * min_ring_size ) / min_normalized_weight ,
static_cast < double > ( max_ring_size ) ) ;
@ -537,7 +513,7 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
double target_hashes = 0.0 ;
uint64_t min_hashes_per_host = ring_size ;
uint64_t max_hashes_per_host = 0 ;
for ( size_t i = 0 ; i < num_subchannels ( ) ; + + i ) {
for ( size_t i = 0 ; i < subchannel_list - > num_subchannels ( ) ; + + i ) {
const std : : string & address_string = address_weights [ i ] . address ;
hash_key_buffer . assign ( address_string . begin ( ) , address_string . end ( ) ) ;
hash_key_buffer . emplace_back ( ' _ ' ) ;
@ -550,7 +526,7 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
absl : : string_view hash_key ( hash_key_buffer . data ( ) ,
hash_key_buffer . size ( ) ) ;
const uint64_t hash = XXH64 ( hash_key . data ( ) , hash_key . size ( ) , 0 ) ;
ring_ . push_back ( { hash , subchannel ( i ) } ) ;
ring_ . push_back ( { hash , i } ) ;
+ + count ;
+ + current_hashes ;
hash_key_buffer . erase ( offset_start , hash_key_buffer . end ( ) ) ;
@ -561,14 +537,34 @@ RingHash::RingHashSubchannelList::RingHashSubchannelList(
std : : max ( static_cast < uint64_t > ( i ) , max_hashes_per_host ) ;
}
std : : sort ( ring_ . begin ( ) , ring_ . end ( ) ,
[ ] ( const RingHashSubchannelList : : RingEntry & lhs ,
const RingHashSubchannelList : : RingEntry & rhs ) - > bool {
[ ] ( const RingEntry & lhs , const RingEntry & rhs ) - > bool {
return lhs . hash < rhs . hash ;
} ) ;
}
//
// RingHash::RingHashSubchannelList
//
RingHash : : RingHashSubchannelList : : RingHashSubchannelList (
RingHash * policy , ServerAddressList addresses , const ChannelArgs & args )
: SubchannelList ( policy ,
( GRPC_TRACE_FLAG_ENABLED ( grpc_lb_ring_hash_trace )
? " RingHashSubchannelList "
: nullptr ) ,
std : : move ( addresses ) , policy - > channel_control_helper ( ) ,
args ) ,
num_idle_ ( num_subchannels ( ) ) {
// Need to maintain a ref to the LB policy as long as we maintain
// any references to subchannels, since the subchannels'
// pollset_sets will include the LB policy's pollset_set.
policy - > Ref ( DEBUG_LOCATION , " subchannel_list " ) . release ( ) ;
// Construct the ring.
ring_ = MakeRefCounted < Ring > ( policy - > config_ . get ( ) , this , args ) ;
if ( GRPC_TRACE_FLAG_ENABLED ( grpc_lb_ring_hash_trace ) ) {
gpr_log ( GPR_INFO ,
" [RH %p] created subchannel list %p with % " PRIuPTR " ring entries " ,
policy , this , ring_ . size ( ) ) ;
policy , this , ring_ - > ring ( ) . size ( ) ) ;
}
}
@ -660,7 +656,7 @@ void RingHash::RingHashSubchannelList::UpdateRingHashConnectivityStateLocked(
// Note that we use our own picker regardless of connectivity state.
p - > channel_control_helper ( ) - > UpdateState (
state , status ,
MakeRefCounted < Picker > ( Ref ( DEBUG_LOCATION , " RingHashPicker " ) ) ) ;
MakeRefCounted < Picker > ( p - > Ref ( DEBUG_LOCATION , " RingHashPicker " ) , this ) ) ;
// While the ring_hash policy is reporting TRANSIENT_FAILURE, it will
// not be getting any pick requests from the priority policy.
// However, because the ring_hash policy does not attempt to
@ -707,7 +703,6 @@ void RingHash::RingHashSubchannelData::ProcessConnectivityChangeLocked(
absl : : optional < grpc_connectivity_state > old_state ,
grpc_connectivity_state new_state ) {
RingHash * p = static_cast < RingHash * > ( subchannel_list ( ) - > policy ( ) ) ;
grpc_connectivity_state last_connectivity_state = GetConnectivityState ( ) ;
if ( GRPC_TRACE_FLAG_ENABLED ( grpc_lb_ring_hash_trace ) ) {
gpr_log (
GPR_INFO ,
@ -715,7 +710,7 @@ void RingHash::RingHashSubchannelData::ProcessConnectivityChangeLocked(
" (index % " PRIuPTR " of % " PRIuPTR " ): prev_state=%s new_state=%s " ,
p , subchannel ( ) , subchannel_list ( ) , Index ( ) ,
subchannel_list ( ) - > num_subchannels ( ) ,
ConnectivityStateName ( last_connectivity_state ) ,
ConnectivityStateName ( logical_connectivity_state_ ) ,
ConnectivityStateName ( new_state ) ) ;
}
GPR_ASSERT ( subchannel ( ) ! = nullptr ) ;
@ -735,34 +730,23 @@ void RingHash::RingHashSubchannelData::ProcessConnectivityChangeLocked(
const bool connection_attempt_complete = new_state ! = GRPC_CHANNEL_CONNECTING ;
// Decide what state to report for the purposes of aggregation and
// picker behavior.
// If the last recorded state was TRANSIENT_FAILURE, ignore the update
// unless the new state is READY.
bool update_status = true ;
absl : : Status status = connectivity_status ( ) ;
if ( last_connectivity_state = = GRPC_CHANNEL_TRANSIENT_FAILURE & &
new_state ! = GRPC_CHANNEL_READY & &
new_state ! = GRPC_CHANNEL_TRANSIENT_FAILURE ) {
new_state = GRPC_CHANNEL_TRANSIENT_FAILURE ;
{
MutexLock lock ( & mu_ ) ;
status = connectivity_status_ ;
}
update_status = false ;
}
// Update state counters used for aggregation.
subchannel_list ( ) - > UpdateStateCountersLocked ( last_connectivity_state ,
new_state ) ;
// Update status seen by picker if needed.
if ( update_status ) {
MutexLock lock ( & mu_ ) ;
connectivity_status_ = connectivity_status ( ) ;
// If the last recorded state was TRANSIENT_FAILURE, ignore the change
// unless the new state is READY (or TF again, in which case we need
// to update the status).
if ( logical_connectivity_state_ ! = GRPC_CHANNEL_TRANSIENT_FAILURE | |
new_state = = GRPC_CHANNEL_READY | |
new_state = = GRPC_CHANNEL_TRANSIENT_FAILURE ) {
// Update state counters used for aggregation.
subchannel_list ( ) - > UpdateStateCountersLocked ( logical_connectivity_state_ ,
new_state ) ;
// Update logical state.
logical_connectivity_state_ = new_state ;
logical_connectivity_status_ = connectivity_status ( ) ;
}
// Update last seen state, also used by picker.
connectivity_state_ . store ( new_state , std : : memory_order_relaxed ) ;
// Update the RH policy's connectivity state, creating new picker and new
// ring.
subchannel_list ( ) - > UpdateRingHashConnectivityStateLocked (
Index ( ) , connection_attempt_complete , status ) ;
Index ( ) , connection_attempt_complete , logical_connectivity_status_ ) ;
}
//
Mark D. Roth
2 years ago
committed by
GitHub