mirror of https://github.com/grpc/grpc.git
[resolver and LB policy APIs] change address list to support multiple addresses per endpoint (#33567)
More changes as part of the dualstack design: - Change resolver and LB policy APIs to support multiple addresses per endpoint. Specifically, replace `ServerAddress` with `EndpointAddresses`, which encodes more than one address. Per-address channel args are retained at the same level, so they are now per-endpoint. For now, `EndpointAddress` provides a single-address ctor and a single-address accessor for backward compatibility, so `ServerAdress` is an alias for `EndpointAddresses`; eventually, this alias and the single-address methods will be removed. - Add an `EndpointAddressSet` class, which represents an unordered set of addresses to be used as a map key. This will be used in a number of LB policies that need to store per-endpoint state. - Change the LB policy API's `ChannelControlHelper::CreateSubchannel()` method to take the address and per-endpoint channel args as separate parameters, so that we don't need to construct a legacy `ServerAddress` object as we create a new subchannel for each address in the endpoint. - Change pick_first to flatten the address list. - Change ring_hash to use `EndpointAddressSet` as the key for its endpoint map, and to use the first address of the endpoint as the hash key. - Change WRR to use `EndpointAddressSet` as the key for its endpoint weight map. Note that support for multiple addresses per endpoint is guarded in RR by the existing `round_robin_delegate_to_pick_fist` experiment and in WRR by the existing `wrr_delegate_to_pick_first` experiment. This PR does *not* include support for multiple addresses per endpoint for the outlier_detection or xds_override_host LB policies; those will come in subsequent PRs.pull/34527/head
Mark D. Roth
1 year ago
committed by
GitHub
parent
d68161a64f
commit
fd2e8c9462
91 changed files with 1316 additions and 678 deletions
@ -0,0 +1,147 @@ |
||||
//
|
||||
//
|
||||
// Copyright 2018 gRPC authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
//
|
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#include "src/core/lib/resolver/endpoint_addresses.h" |
||||
|
||||
#include <string.h> |
||||
|
||||
#include <algorithm> |
||||
#include <string> |
||||
#include <utility> |
||||
#include <vector> |
||||
|
||||
#include "absl/status/status.h" |
||||
#include "absl/status/statusor.h" |
||||
#include "absl/strings/str_cat.h" |
||||
#include "absl/strings/str_join.h" |
||||
|
||||
#include <grpc/support/log.h> |
||||
|
||||
#include "src/core/lib/address_utils/sockaddr_utils.h" |
||||
#include "src/core/lib/channel/channel_args.h" |
||||
#include "src/core/lib/gpr/useful.h" |
||||
|
||||
// IWYU pragma: no_include <sys/socket.h>
|
||||
|
||||
namespace grpc_core { |
||||
|
||||
EndpointAddresses::EndpointAddresses(const grpc_resolved_address& address, |
||||
const ChannelArgs& args) |
||||
: addresses_(1, address), args_(args) {} |
||||
|
||||
EndpointAddresses::EndpointAddresses( |
||||
std::vector<grpc_resolved_address> addresses, const ChannelArgs& args) |
||||
: addresses_(std::move(addresses)), args_(args) { |
||||
GPR_ASSERT(!addresses_.empty()); |
||||
} |
||||
|
||||
EndpointAddresses::EndpointAddresses(const EndpointAddresses& other) |
||||
: addresses_(other.addresses_), args_(other.args_) {} |
||||
|
||||
EndpointAddresses& EndpointAddresses::operator=( |
||||
const EndpointAddresses& other) { |
||||
if (&other == this) return *this; |
||||
addresses_ = other.addresses_; |
||||
args_ = other.args_; |
||||
return *this; |
||||
} |
||||
|
||||
EndpointAddresses::EndpointAddresses(EndpointAddresses&& other) noexcept |
||||
: addresses_(std::move(other.addresses_)), args_(std::move(other.args_)) {} |
||||
|
||||
EndpointAddresses& EndpointAddresses::operator=( |
||||
EndpointAddresses&& other) noexcept { |
||||
addresses_ = std::move(other.addresses_); |
||||
args_ = std::move(other.args_); |
||||
return *this; |
||||
} |
||||
|
||||
int EndpointAddresses::Cmp(const EndpointAddresses& other) const { |
||||
for (size_t i = 0; i < addresses_.size(); ++i) { |
||||
if (other.addresses_.size() == i) return 1; |
||||
if (addresses_[i].len > other.addresses_[i].len) return 1; |
||||
if (addresses_[i].len < other.addresses_[i].len) return -1; |
||||
int retval = |
||||
memcmp(addresses_[i].addr, other.addresses_[i].addr, addresses_[i].len); |
||||
if (retval != 0) return retval; |
||||
} |
||||
if (other.addresses_.size() > addresses_.size()) return -1; |
||||
return QsortCompare(args_, other.args_); |
||||
} |
||||
|
||||
std::string EndpointAddresses::ToString() const { |
||||
std::vector<std::string> addr_strings; |
||||
for (const auto& address : addresses_) { |
||||
auto addr_str = grpc_sockaddr_to_string(&address, false); |
||||
addr_strings.push_back(addr_str.ok() ? std::move(*addr_str) |
||||
: addr_str.status().ToString()); |
||||
} |
||||
std::vector<std::string> parts = { |
||||
absl::StrCat("addrs=[", absl::StrJoin(addr_strings, ", "), "]")}; |
||||
if (args_ != ChannelArgs()) { |
||||
parts.emplace_back(absl::StrCat("args=", args_.ToString())); |
||||
} |
||||
return absl::StrJoin(parts, " "); |
||||
} |
||||
|
||||
bool EndpointAddressSet::operator==(const EndpointAddressSet& other) const { |
||||
if (addresses_.size() != other.addresses_.size()) return false; |
||||
auto other_it = other.addresses_.begin(); |
||||
for (auto it = addresses_.begin(); it != addresses_.end(); ++it) { |
||||
GPR_ASSERT(other_it != other.addresses_.end()); |
||||
if (it->len != other_it->len || |
||||
memcmp(it->addr, other_it->addr, it->len) != 0) { |
||||
return false; |
||||
} |
||||
++other_it; |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
bool EndpointAddressSet::operator<(const EndpointAddressSet& other) const { |
||||
auto other_it = other.addresses_.begin(); |
||||
for (auto it = addresses_.begin(); it != addresses_.end(); ++it) { |
||||
if (other_it == other.addresses_.end()) return true; |
||||
if (it->len < other_it->len) return true; |
||||
if (it->len > other_it->len) return false; |
||||
int r = memcmp(it->addr, other_it->addr, it->len); |
||||
if (r != 0) return r < 0; |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
bool EndpointAddressSet::ResolvedAddressLessThan::operator()( |
||||
const grpc_resolved_address& addr1, |
||||
const grpc_resolved_address& addr2) const { |
||||
if (addr1.len < addr2.len) return true; |
||||
return memcmp(addr1.addr, addr2.addr, addr1.len) < 0; |
||||
} |
||||
|
||||
std::string EndpointAddressSet::ToString() const { |
||||
std::vector<std::string> parts; |
||||
parts.reserve(addresses_.size()); |
||||
for (const auto& address : addresses_) { |
||||
parts.push_back( |
||||
grpc_sockaddr_to_string(&address, false).value_or("<unknown>")); |
||||
} |
||||
return absl::StrCat("{", absl::StrJoin(parts, ", "), "}"); |
||||
} |
||||
|
||||
} // namespace grpc_core
|
||||
@ -0,0 +1,116 @@ |
||||
//
|
||||
//
|
||||
// Copyright 2018 gRPC authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
//
|
||||
|
||||
#ifndef GRPC_SRC_CORE_LIB_RESOLVER_ENDPOINT_ADDRESSES_H |
||||
#define GRPC_SRC_CORE_LIB_RESOLVER_ENDPOINT_ADDRESSES_H |
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#include <set> |
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
#include "src/core/lib/channel/channel_args.h" |
||||
#include "src/core/lib/iomgr/resolved_address.h" |
||||
|
||||
// A channel arg key prefix used for args that are intended to be used
|
||||
// only internally to resolvers and LB policies and should not be part
|
||||
// of the subchannel key. The channel will automatically filter out any
|
||||
// args with this prefix from the subchannel's args.
|
||||
#define GRPC_ARG_NO_SUBCHANNEL_PREFIX "grpc.internal.no_subchannel." |
||||
|
||||
// A channel arg indicating the weight of an address.
|
||||
#define GRPC_ARG_ADDRESS_WEIGHT GRPC_ARG_NO_SUBCHANNEL_PREFIX "address.weight" |
||||
|
||||
namespace grpc_core { |
||||
|
||||
// A list of addresses for a given endpoint with an associated set of channel
|
||||
// args. Any args present here will be merged into the channel args when a
|
||||
// subchannel is created for each address.
|
||||
class EndpointAddresses { |
||||
public: |
||||
// For backward compatibility.
|
||||
// TODO(roth): Remove when callers have been updated.
|
||||
EndpointAddresses(const grpc_resolved_address& address, |
||||
const ChannelArgs& args); |
||||
|
||||
// addresses must not be empty.
|
||||
EndpointAddresses(std::vector<grpc_resolved_address> addresses, |
||||
const ChannelArgs& args); |
||||
|
||||
// Copyable.
|
||||
EndpointAddresses(const EndpointAddresses& other); |
||||
EndpointAddresses& operator=(const EndpointAddresses& other); |
||||
|
||||
// Movable.
|
||||
EndpointAddresses(EndpointAddresses&& other) noexcept; |
||||
EndpointAddresses& operator=(EndpointAddresses&& other) noexcept; |
||||
|
||||
bool operator==(const EndpointAddresses& other) const { |
||||
return Cmp(other) == 0; |
||||
} |
||||
bool operator<(const EndpointAddresses& other) const { |
||||
return Cmp(other) < 0; |
||||
} |
||||
|
||||
int Cmp(const EndpointAddresses& other) const; |
||||
|
||||
// For backward compatibility only.
|
||||
// TODO(roth): Remove when all callers have been updated.
|
||||
const grpc_resolved_address& address() const { return addresses_[0]; } |
||||
|
||||
const std::vector<grpc_resolved_address>& addresses() const { |
||||
return addresses_; |
||||
} |
||||
const ChannelArgs& args() const { return args_; } |
||||
|
||||
// TODO(ctiller): Prior to making this a public API we should ensure that the
|
||||
// channel args are not part of the generated string, lest we make that debug
|
||||
// format load-bearing via Hyrum's law.
|
||||
std::string ToString() const; |
||||
|
||||
private: |
||||
std::vector<grpc_resolved_address> addresses_; |
||||
ChannelArgs args_; |
||||
}; |
||||
|
||||
using EndpointAddressesList = std::vector<EndpointAddresses>; |
||||
|
||||
class EndpointAddressSet { |
||||
public: |
||||
explicit EndpointAddressSet( |
||||
const std::vector<grpc_resolved_address>& addresses) |
||||
: addresses_(addresses.begin(), addresses.end()) {} |
||||
|
||||
bool operator==(const EndpointAddressSet& other) const; |
||||
bool operator<(const EndpointAddressSet& other) const; |
||||
|
||||
std::string ToString() const; |
||||
|
||||
private: |
||||
struct ResolvedAddressLessThan { |
||||
bool operator()(const grpc_resolved_address& addr1, |
||||
const grpc_resolved_address& addr2) const; |
||||
}; |
||||
|
||||
std::set<grpc_resolved_address, ResolvedAddressLessThan> addresses_; |
||||
}; |
||||
|
||||
} // namespace grpc_core
|
||||
|
||||
#endif // GRPC_SRC_CORE_LIB_RESOLVER_ENDPOINT_ADDRESSES_H
|
||||
@ -1,89 +0,0 @@ |
||||
//
|
||||
//
|
||||
// Copyright 2018 gRPC authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
//
|
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#include "src/core/lib/resolver/server_address.h" |
||||
|
||||
#include <string.h> |
||||
|
||||
#include <algorithm> |
||||
#include <string> |
||||
#include <utility> |
||||
#include <vector> |
||||
|
||||
#include "absl/status/status.h" |
||||
#include "absl/status/statusor.h" |
||||
#include "absl/strings/str_cat.h" |
||||
#include "absl/strings/str_join.h" |
||||
|
||||
#include "src/core/lib/address_utils/sockaddr_utils.h" |
||||
#include "src/core/lib/channel/channel_args.h" |
||||
#include "src/core/lib/gpr/useful.h" |
||||
|
||||
// IWYU pragma: no_include <sys/socket.h>
|
||||
|
||||
namespace grpc_core { |
||||
|
||||
//
|
||||
// ServerAddress
|
||||
//
|
||||
|
||||
ServerAddress::ServerAddress(const grpc_resolved_address& address, |
||||
const ChannelArgs& args) |
||||
: address_(address), args_(args) {} |
||||
|
||||
ServerAddress::ServerAddress(const ServerAddress& other) |
||||
: address_(other.address_), args_(other.args_) {} |
||||
|
||||
ServerAddress& ServerAddress::operator=(const ServerAddress& other) { |
||||
if (&other == this) return *this; |
||||
address_ = other.address_; |
||||
args_ = other.args_; |
||||
return *this; |
||||
} |
||||
|
||||
ServerAddress::ServerAddress(ServerAddress&& other) noexcept |
||||
: address_(other.address_), args_(std::move(other.args_)) {} |
||||
|
||||
ServerAddress& ServerAddress::operator=(ServerAddress&& other) noexcept { |
||||
address_ = other.address_; |
||||
args_ = std::move(other.args_); |
||||
return *this; |
||||
} |
||||
|
||||
int ServerAddress::Cmp(const ServerAddress& other) const { |
||||
if (address_.len > other.address_.len) return 1; |
||||
if (address_.len < other.address_.len) return -1; |
||||
int retval = memcmp(address_.addr, other.address_.addr, address_.len); |
||||
if (retval != 0) return retval; |
||||
return QsortCompare(args_, other.args_); |
||||
} |
||||
|
||||
std::string ServerAddress::ToString() const { |
||||
auto addr_str = grpc_sockaddr_to_string(&address_, false); |
||||
std::vector<std::string> parts = { |
||||
addr_str.ok() ? addr_str.value() : addr_str.status().ToString(), |
||||
}; |
||||
if (args_ != ChannelArgs()) { |
||||
parts.emplace_back(absl::StrCat("args=", args_.ToString())); |
||||
} |
||||
return absl::StrJoin(parts, " "); |
||||
} |
||||
|
||||
} // namespace grpc_core
|
||||
Loading…
Reference in new issue