Refactor channel pool

pull/16190/head
Muxi Yan 6 years ago
parent d72d5b2c8e
commit 37dbad80d5
  1. 4
      src/objective-c/GRPCClient/GRPCCall+ChannelArg.m
  2. 69
      src/objective-c/GRPCClient/private/GRPCChannel.h
  3. 379
      src/objective-c/GRPCClient/private/GRPCChannel.m
  4. 55
      src/objective-c/GRPCClient/private/GRPCChannelPool.h
  5. 209
      src/objective-c/GRPCClient/private/GRPCChannelPool.m
  6. 16
      src/objective-c/GRPCClient/private/GRPCWrappedCall.m
  7. 161
      src/objective-c/tests/ChannelTests/ChannelPoolTest.m
  8. 97
      src/objective-c/tests/ChannelTests/ChannelTests.m

@ -18,7 +18,7 @@
#import "GRPCCall+ChannelArg.h"
#import "private/GRPCChannel.h"
#import "private/GRPCChannelPool.h"
#import "private/GRPCHost.h"
#import <grpc/impl/codegen/compression_types.h>
@ -36,7 +36,7 @@
}
+ (void)closeOpenConnections {
[GRPCChannel closeOpenConnections];
[GRPCChannelPool closeOpenConnections];
}
+ (void)setDefaultCompressMethod:(GRPCCompressAlgorithm)algorithm forhost:(nonnull NSString *)host {

@ -20,11 +20,37 @@
#include <grpc/grpc.h>
@protocol GRPCChannelFactory;
@class GRPCCompletionQueue;
@class GRPCCallOptions;
@class GRPCChannelConfiguration;
struct grpc_channel_credentials;
NS_ASSUME_NONNULL_BEGIN
/** Caching signature of a channel. */
@interface GRPCChannelConfiguration : NSObject<NSCopying>
/** The host that this channel is connected to. */
@property(copy, readonly) NSString *host;
/**
* Options of the corresponding call. Note that only the channel-related options are of interest to
* this class.
*/
@property(strong, readonly) GRPCCallOptions *callOptions;
/** Acquire the factory to generate a new channel with current configurations. */
@property(readonly) id<GRPCChannelFactory> channelFactory;
/** Acquire the dictionary of channel args with current configurations. */
@property(copy, readonly) NSDictionary *channelArgs;
- (nullable instancetype)initWithHost:(NSString *)host callOptions:(GRPCCallOptions *)callOptions;
@end
/**
* Each separate instance of this class represents at least one TCP connection to the provided host.
*/
@ -35,40 +61,45 @@ struct grpc_channel_credentials;
+ (nullable instancetype) new NS_UNAVAILABLE;
/**
* Returns a channel connecting to \a host with options as \a callOptions. The channel may be new
* or a cached channel that is already connected.
* Create a channel with remote \a host and signature \a channelConfigurations. Destroy delay is
* defaulted to 30 seconds.
*/
+ (nullable instancetype)channelWithHost:(nonnull NSString *)host
callOptions:(nullable GRPCCallOptions *)callOptions;
- (nullable instancetype)initWithChannelConfiguration:(GRPCChannelConfiguration *)channelConfiguration;
/**
* Create a channel object with the signature \a config.
* Create a channel with remote \a host, signature \a channelConfigurations, and destroy delay of
* \a destroyDelay.
*/
+ (nullable instancetype)createChannelWithConfiguration:(nonnull GRPCChannelConfiguration *)config;
- (nullable instancetype)initWithChannelConfiguration:(GRPCChannelConfiguration *)channelConfiguration
destroyDelay:(NSTimeInterval)destroyDelay NS_DESIGNATED_INITIALIZER;
/**
* Get a grpc core call object from this channel.
* Create a grpc core call object from this channel. The channel's refcount is added by 1. If no
* call is created, NULL is returned, and if the reason is because the channel is already
* disconnected, \a disconnected is set to YES. When the returned call is unreffed, the caller is
* obligated to call \a unref method once. \a disconnected may be null.
*/
- (nullable grpc_call *)unmanagedCallWithPath:(nonnull NSString *)path
completionQueue:(nonnull GRPCCompletionQueue *)queue
callOptions:(nonnull GRPCCallOptions *)callOptions;
- (nullable grpc_call *)unmanagedCallWithPath:(NSString *)path
completionQueue:(GRPCCompletionQueue *)queue
callOptions:(GRPCCallOptions *)callOptions
disconnected:(BOOL * _Nullable)disconnected;
/**
* Increase the refcount of the channel. If the channel was timed to be destroyed, cancel the timer.
* Unref the channel when a call is done. It also decreases the channel's refcount. If the refcount
* of the channel decreases to 0, the channel is destroyed after the destroy delay.
*/
- (void)ref;
- (void)unref;
/**
* Decrease the refcount of the channel. If the refcount of the channel decrease to 0, the channel
* is destroyed after 30 seconds.
* Force the channel to be disconnected and destroyed.
*/
- (void)unref;
- (void)disconnect;
/**
* Force the channel to be disconnected and destroyed immediately.
* Return whether the channel is already disconnected.
*/
- (void)disconnect;
@property(readonly) BOOL disconnected;
// TODO (mxyan): deprecate with GRPCCall:closeOpenConnections
+ (void)closeOpenConnections;
@end
NS_ASSUME_NONNULL_END

@ -28,139 +28,220 @@
#import "GRPCInsecureChannelFactory.h"
#import "GRPCSecureChannelFactory.h"
#import "version.h"
#import "../internal/GRPCCallOptions+Internal.h"
#import <GRPCClient/GRPCCall+Cronet.h>
#import <GRPCClient/GRPCCallOptions.h>
/** When all calls of a channel are destroyed, destroy the channel after this much seconds. */
NSTimeInterval kChannelDestroyDelay = 30;
NSTimeInterval kDefaultChannelDestroyDelay = 30;
/** Global instance of channel pool. */
static GRPCChannelPool *gChannelPool;
@implementation GRPCChannelConfiguration
/**
* Time the channel destroy when the channel's calls are unreffed. If there's new call, reset the
* timer.
*/
@interface GRPCChannelRef : NSObject
- (instancetype)initWithDestroyDelay:(NSTimeInterval)destroyDelay
destroyChannelCallback:(void (^)())destroyChannelCallback;
- (nullable instancetype)initWithHost:(NSString *)host callOptions:(GRPCCallOptions *)callOptions {
NSAssert(host.length, @"Host must not be empty.");
NSAssert(callOptions, @"callOptions must not be empty.");
if ((self = [super init])) {
_host = [host copy];
_callOptions = [callOptions copy];
}
return self;
}
/** Add call ref count to the channel and maybe reset the timer. */
- (void)refChannel;
- (id<GRPCChannelFactory>)channelFactory {
NSError *error;
id<GRPCChannelFactory> factory;
GRPCTransportType type = _callOptions.transportType;
switch (type) {
case GRPCTransportTypeChttp2BoringSSL:
// TODO (mxyan): Remove when the API is deprecated
#ifdef GRPC_COMPILE_WITH_CRONET
if (![GRPCCall isUsingCronet]) {
#endif
factory = [GRPCSecureChannelFactory
factoryWithPEMRootCertificates:_callOptions.PEMRootCertificates
privateKey:_callOptions.PEMPrivateKey
certChain:_callOptions.PEMCertChain
error:&error];
if (factory == nil) {
NSLog(@"Error creating secure channel factory: %@", error);
}
return factory;
#ifdef GRPC_COMPILE_WITH_CRONET
}
#endif
// fallthrough
case GRPCTransportTypeCronet:
return [GRPCCronetChannelFactory sharedInstance];
case GRPCTransportTypeInsecure:
return [GRPCInsecureChannelFactory sharedInstance];
}
}
/** Reduce call ref count to the channel and maybe set the timer. */
- (void)unrefChannel;
- (NSDictionary *)channelArgs {
NSMutableDictionary *args = [NSMutableDictionary new];
/** Disconnect the channel. Any further ref/unref are discarded. */
- (void)disconnect;
NSString *userAgent = @"grpc-objc/" GRPC_OBJC_VERSION_STRING;
NSString *userAgentPrefix = _callOptions.userAgentPrefix;
if (userAgentPrefix) {
args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] =
[_callOptions.userAgentPrefix stringByAppendingFormat:@" %@", userAgent];
} else {
args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] = userAgent;
}
@end
NSString *hostNameOverride = _callOptions.hostNameOverride;
if (hostNameOverride) {
args[@GRPC_SSL_TARGET_NAME_OVERRIDE_ARG] = hostNameOverride;
}
@implementation GRPCChannelRef {
NSTimeInterval _destroyDelay;
void (^_destroyChannelCallback)();
if (_callOptions.responseSizeLimit) {
args[@GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH] =
[NSNumber numberWithUnsignedInteger:_callOptions.responseSizeLimit];
}
NSUInteger _refCount;
BOOL _disconnected;
dispatch_queue_t _dispatchQueue;
if (_callOptions.compressionAlgorithm != GRPC_COMPRESS_NONE) {
args[@GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM] =
[NSNumber numberWithInt:_callOptions.compressionAlgorithm];
}
/**
* Date and time when last timer is scheduled. If a firing timer's scheduled date is different
* from this, it is discarded.
*/
NSDate *_lastDispatch;
}
if (_callOptions.keepaliveInterval != 0) {
args[@GRPC_ARG_KEEPALIVE_TIME_MS] =
[NSNumber numberWithUnsignedInteger:(NSUInteger)(_callOptions.keepaliveInterval * 1000)];
args[@GRPC_ARG_KEEPALIVE_TIMEOUT_MS] =
[NSNumber numberWithUnsignedInteger:(NSUInteger)(_callOptions.keepaliveTimeout * 1000)];
}
- (instancetype)initWithDestroyDelay:(NSTimeInterval)destroyDelay
destroyChannelCallback:(void (^)())destroyChannelCallback {
if ((self = [super init])) {
_destroyDelay = destroyDelay;
_destroyChannelCallback = destroyChannelCallback;
if (_callOptions.retryEnabled == NO) {
args[@GRPC_ARG_ENABLE_RETRIES] = [NSNumber numberWithInt:_callOptions.retryEnabled];
}
_refCount = 1;
_disconnected = NO;
if (@available(iOS 8.0, *)) {
_dispatchQueue = dispatch_queue_create(
NULL,
dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_DEFAULT, -1));
} else {
_dispatchQueue = dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL);
if (_callOptions.connectMinTimeout > 0) {
args[@GRPC_ARG_MIN_RECONNECT_BACKOFF_MS] =
[NSNumber numberWithUnsignedInteger:(NSUInteger)(_callOptions.connectMinTimeout * 1000)];
}
_lastDispatch = nil;
if (_callOptions.connectInitialBackoff > 0) {
args[@GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS] = [NSNumber
numberWithUnsignedInteger:(NSUInteger)(_callOptions.connectInitialBackoff * 1000)];
}
return self;
}
- (void)refChannel {
dispatch_async(_dispatchQueue, ^{
if (!self->_disconnected) {
self->_refCount++;
self->_lastDispatch = nil;
if (_callOptions.connectMaxBackoff > 0) {
args[@GRPC_ARG_MAX_RECONNECT_BACKOFF_MS] =
[NSNumber numberWithUnsignedInteger:(NSUInteger)(_callOptions.connectMaxBackoff * 1000)];
}
});
}
- (void)unrefChannel {
dispatch_async(_dispatchQueue, ^{
if (!self->_disconnected) {
self->_refCount--;
if (self->_refCount == 0) {
NSDate *now = [NSDate date];
self->_lastDispatch = now;
dispatch_time_t delay =
dispatch_time(DISPATCH_TIME_NOW, (int64_t)self->_destroyDelay * NSEC_PER_SEC);
dispatch_after(delay, self->_dispatchQueue, ^{
[self timedDisconnectWithScheduleDate:now];
});
if (_callOptions.logContext != nil) {
args[@GRPC_ARG_MOBILE_LOG_CONTEXT] = _callOptions.logContext;
}
if (_callOptions.channelPoolDomain.length != 0) {
args[@GRPC_ARG_CHANNEL_POOL_DOMAIN] = _callOptions.channelPoolDomain;
}
});
[args addEntriesFromDictionary:_callOptions.additionalChannelArgs];
return args;
}
- (void)disconnect {
dispatch_async(_dispatchQueue, ^{
if (!self->_disconnected) {
self->_lastDispatch = nil;
self->_disconnected = YES;
// Break retain loop
self->_destroyChannelCallback = nil;
}
});
- (nonnull id)copyWithZone:(nullable NSZone *)zone {
GRPCChannelConfiguration *newConfig =
[[GRPCChannelConfiguration alloc] initWithHost:_host callOptions:_callOptions];
return newConfig;
}
- (void)timedDisconnectWithScheduleDate:(NSDate *)scheduleDate {
dispatch_async(_dispatchQueue, ^{
if (self->_disconnected || self->_lastDispatch != scheduleDate) {
return;
- (BOOL)isEqual:(id)object {
if (![object isKindOfClass:[GRPCChannelConfiguration class]]) {
return NO;
}
self->_lastDispatch = nil;
self->_disconnected = YES;
self->_destroyChannelCallback();
// Break retain loop
self->_destroyChannelCallback = nil;
});
GRPCChannelConfiguration *obj = (GRPCChannelConfiguration *)object;
if (!(obj.host == _host || (_host != nil && [obj.host isEqualToString:_host]))) return NO;
if (!(obj.callOptions == _callOptions || [obj.callOptions hasChannelOptionsEqualTo:_callOptions]))
return NO;
return YES;
}
- (NSUInteger)hash {
NSUInteger result = 0;
result ^= _host.hash;
result ^= _callOptions.channelOptionsHash;
return result;
}
@end
@implementation GRPCChannel {
GRPCChannelConfiguration *_configuration;
grpc_channel *_unmanagedChannel;
GRPCChannelRef *_channelRef;
dispatch_queue_t _dispatchQueue;
grpc_channel *_unmanagedChannel;
NSTimeInterval _destroyDelay;
NSUInteger _refcount;
NSDate *_lastDispatch;
}
@synthesize disconnected = _disconnected;
- (nullable instancetype)initWithChannelConfiguration:(GRPCChannelConfiguration *)channelConfiguration {
return [self initWithChannelConfiguration:channelConfiguration
destroyDelay:kDefaultChannelDestroyDelay];
}
- (nullable instancetype)initWithChannelConfiguration:(GRPCChannelConfiguration *)channelConfiguration
destroyDelay:(NSTimeInterval)destroyDelay {
NSAssert(channelConfiguration, @"channelConfiguration must not be empty.");
NSAssert(destroyDelay > 0, @"destroyDelay must be greater than 0.");
if ((self = [super init])) {
_configuration = [channelConfiguration copy];
if (@available(iOS 8.0, *)) {
_dispatchQueue = dispatch_queue_create(
NULL,
dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_DEFAULT, -1));
} else {
_dispatchQueue = dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL);
}
// Create gRPC core channel object.
NSString *host = channelConfiguration.host;
NSAssert(host.length != 0, @"host cannot be nil");
NSDictionary *channelArgs;
if (channelConfiguration.callOptions.additionalChannelArgs.count != 0) {
NSMutableDictionary *args = [channelConfiguration.channelArgs mutableCopy];
[args addEntriesFromDictionary:channelConfiguration.callOptions.additionalChannelArgs];
channelArgs = args;
} else {
channelArgs = channelConfiguration.channelArgs;
}
id<GRPCChannelFactory> factory = channelConfiguration.channelFactory;
_unmanagedChannel = [factory createChannelWithHost:host channelArgs:channelArgs];
if (_unmanagedChannel == NULL) {
NSLog(@"Unable to create channel.");
return nil;
}
_destroyDelay = destroyDelay;
_disconnected = NO;
}
return self;
}
- (grpc_call *)unmanagedCallWithPath:(NSString *)path
completionQueue:(GRPCCompletionQueue *)queue
callOptions:(GRPCCallOptions *)callOptions {
callOptions:(GRPCCallOptions *)callOptions
disconnected:(BOOL *)disconnected {
NSAssert(path.length, @"path must not be empty.");
NSAssert(queue, @"completionQueue must not be empty.");
NSAssert(callOptions, @"callOptions must not be empty.");
__block grpc_call *call = nil;
__block BOOL isDisconnected = NO;
__block grpc_call *call = NULL;
dispatch_sync(_dispatchQueue, ^{
if (self->_unmanagedChannel) {
if (self->_disconnected) {
isDisconnected = YES;
} else {
NSAssert(self->_unmanagedChannel != NULL, @"Invalid channel.");
NSString *serverAuthority =
callOptions.transportType == GRPCTransportTypeCronet ? nil : callOptions.serverAuthority;
NSTimeInterval timeout = callOptions.timeout;
@ -182,71 +263,64 @@ static GRPCChannelPool *gChannelPool;
grpc_slice_unref(host_slice);
}
grpc_slice_unref(path_slice);
if (call == NULL) {
NSLog(@"Unable to create call.");
} else {
NSAssert(self->_unmanagedChannel != nil, @"Invalid channeg.");
// Ref the channel;
[self ref];
}
}
});
if (disconnected != nil) {
*disconnected = isDisconnected;
}
return call;
}
// This function should be called on _dispatchQueue.
- (void)ref {
dispatch_async(_dispatchQueue, ^{
if (self->_unmanagedChannel) {
[self->_channelRef refChannel];
_refcount++;
if (_refcount == 1 && _lastDispatch != nil) {
_lastDispatch = nil;
}
});
}
- (void)unref {
dispatch_async(_dispatchQueue, ^{
if (self->_unmanagedChannel) {
[self->_channelRef unrefChannel];
self->_refcount--;
if (self->_refcount == 0 && !self->_disconnected) {
// Start timer.
dispatch_time_t delay =
dispatch_time(DISPATCH_TIME_NOW, (int64_t)self->_destroyDelay * NSEC_PER_SEC);
NSDate *now = [NSDate date];
self->_lastDispatch = now;
dispatch_after(delay, self->_dispatchQueue, ^{
if (self->_lastDispatch == now) {
grpc_channel_destroy(self->_unmanagedChannel);
self->_unmanagedChannel = NULL;
self->_disconnected = YES;
}
});
}
- (void)disconnect {
dispatch_async(_dispatchQueue, ^{
if (self->_unmanagedChannel) {
grpc_channel_destroy(self->_unmanagedChannel);
self->_unmanagedChannel = nil;
[self->_channelRef disconnect];
}
});
}
- (void)destroyChannel {
- (void)disconnect {
dispatch_async(_dispatchQueue, ^{
if (self->_unmanagedChannel) {
if (!self->_disconnected) {
grpc_channel_destroy(self->_unmanagedChannel);
self->_unmanagedChannel = nil;
[gChannelPool removeChannel:self];
self->_disconnected = YES;
}
});
}
- (nullable instancetype)initWithUnmanagedChannel:(grpc_channel *_Nullable)unmanagedChannel
configuration:(GRPCChannelConfiguration *)configuration {
NSAssert(configuration, @"Configuration must not be empty.");
if (!unmanagedChannel) {
return nil;
}
if ((self = [super init])) {
_unmanagedChannel = unmanagedChannel;
_configuration = [configuration copy];
_channelRef = [[GRPCChannelRef alloc] initWithDestroyDelay:kChannelDestroyDelay
destroyChannelCallback:^{
[self destroyChannel];
}];
if (@available(iOS 8.0, *)) {
_dispatchQueue = dispatch_queue_create(
NULL,
dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_DEFAULT, -1));
} else {
_dispatchQueue = dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL);
}
}
return self;
- (BOOL)disconnected {
__block BOOL disconnected;
dispatch_sync(_dispatchQueue, ^{
disconnected = self->_disconnected;
});
return disconnected;
}
- (void)dealloc {
@ -255,47 +329,4 @@ static GRPCChannelPool *gChannelPool;
}
}
+ (nullable instancetype)createChannelWithConfiguration:(GRPCChannelConfiguration *)config {
NSAssert(config != nil, @"configuration cannot be empty");
NSString *host = config.host;
NSAssert(host.length != 0, @"host cannot be nil");
NSDictionary *channelArgs;
if (config.callOptions.additionalChannelArgs.count != 0) {
NSMutableDictionary *args = [config.channelArgs mutableCopy];
[args addEntriesFromDictionary:config.callOptions.additionalChannelArgs];
channelArgs = args;
} else {
channelArgs = config.channelArgs;
}
id<GRPCChannelFactory> factory = config.channelFactory;
grpc_channel *unmanaged_channel = [factory createChannelWithHost:host channelArgs:channelArgs];
return [[GRPCChannel alloc] initWithUnmanagedChannel:unmanaged_channel configuration:config];
}
+ (nullable instancetype)channelWithHost:(NSString *)host
callOptions:(GRPCCallOptions *)callOptions {
static dispatch_once_t initChannelPool;
dispatch_once(&initChannelPool, ^{
gChannelPool = [[GRPCChannelPool alloc] init];
});
NSURL *hostURL = [NSURL URLWithString:[@"https://" stringByAppendingString:host]];
if (hostURL.host && !hostURL.port) {
host = [hostURL.host stringByAppendingString:@":443"];
}
GRPCChannelConfiguration *channelConfig =
[[GRPCChannelConfiguration alloc] initWithHost:host callOptions:callOptions];
if (channelConfig == nil) {
return nil;
}
return [gChannelPool channelWithConfiguration:channelConfig];
}
+ (void)closeOpenConnections {
[gChannelPool removeAndCloseAllChannels];
}
@end

@ -29,48 +29,45 @@ NS_ASSUME_NONNULL_BEGIN
@class GRPCChannel;
/** Caching signature of a channel. */
@interface GRPCChannelConfiguration : NSObject<NSCopying>
/** The host that this channel is connected to. */
@property(copy, readonly) NSString *host;
/**
* Options of the corresponding call. Note that only the channel-related options are of interest to
* this class.
*/
@property(strong, readonly) GRPCCallOptions *callOptions;
/** Acquire the factory to generate a new channel with current configurations. */
@property(readonly) id<GRPCChannelFactory> channelFactory;
/** Acquire the dictionary of channel args with current configurations. */
@property(copy, readonly) NSDictionary *channelArgs;
- (nullable instancetype)initWithHost:(NSString *)host callOptions:(GRPCCallOptions *)callOptions;
@end
/**
* Manage the pool of connected channels. When a channel is no longer referenced by any call,
* destroy the channel after a certain period of time elapsed.
*/
@interface GRPCChannelPool : NSObject
/**
* Get the singleton instance
*/
+ (nullable instancetype)sharedInstance;
/**
* Return a channel with a particular configuration. If the channel does not exist, execute \a
* createChannel then add it in the pool. If the channel exists, increase its reference count.
*/
- (GRPCChannel *)channelWithConfiguration:(GRPCChannelConfiguration *)configuration;
- (GRPCChannel *)channelWithHost:(NSString *)host
callOptions:(GRPCCallOptions *)callOptions;
/**
* This method is deprecated.
*
* Destroy all open channels and close their connections.
*/
+ (void)closeOpenConnections;
/** Remove a channel from the pool. */
- (void)removeChannel:(GRPCChannel *)channel;
// Test-only methods below
/** Clear all channels in the pool. */
- (void)removeAllChannels;
/**
* Return a channel with a special destroy delay. If \a destroyDelay is 0, use the default destroy
* delay.
*/
- (GRPCChannel *)channelWithHost:(NSString *)host
callOptions:(GRPCCallOptions *)callOptions
destroyDelay:(NSTimeInterval)destroyDelay;
/** Clear all channels in the pool and destroy the channels. */
- (void)removeAndCloseAllChannels;
/**
* Simulate a network transition event and destroy all channels.
*/
- (void)destroyAllChannels;
@end

@ -33,145 +33,21 @@
extern const char *kCFStreamVarName;
@implementation GRPCChannelConfiguration
static GRPCChannelPool *gChannelPool;
static dispatch_once_t gInitChannelPool;
- (nullable instancetype)initWithHost:(NSString *)host callOptions:(GRPCCallOptions *)callOptions {
NSAssert(host.length, @"Host must not be empty.");
NSAssert(callOptions, @"callOptions must not be empty.");
if ((self = [super init])) {
_host = [host copy];
_callOptions = [callOptions copy];
}
return self;
}
- (id<GRPCChannelFactory>)channelFactory {
NSError *error;
id<GRPCChannelFactory> factory;
GRPCTransportType type = _callOptions.transportType;
switch (type) {
case GRPCTransportTypeChttp2BoringSSL:
// TODO (mxyan): Remove when the API is deprecated
#ifdef GRPC_COMPILE_WITH_CRONET
if (![GRPCCall isUsingCronet]) {
#endif
factory = [GRPCSecureChannelFactory
factoryWithPEMRootCertificates:_callOptions.PEMRootCertificates
privateKey:_callOptions.PEMPrivateKey
certChain:_callOptions.PEMCertChain
error:&error];
if (factory == nil) {
NSLog(@"Error creating secure channel factory: %@", error);
}
return factory;
#ifdef GRPC_COMPILE_WITH_CRONET
}
#endif
// fallthrough
case GRPCTransportTypeCronet:
return [GRPCCronetChannelFactory sharedInstance];
case GRPCTransportTypeInsecure:
return [GRPCInsecureChannelFactory sharedInstance];
}
}
- (NSDictionary *)channelArgs {
NSMutableDictionary *args = [NSMutableDictionary new];
NSString *userAgent = @"grpc-objc/" GRPC_OBJC_VERSION_STRING;
NSString *userAgentPrefix = _callOptions.userAgentPrefix;
if (userAgentPrefix) {
args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] =
[_callOptions.userAgentPrefix stringByAppendingFormat:@" %@", userAgent];
} else {
args[@GRPC_ARG_PRIMARY_USER_AGENT_STRING] = userAgent;
}
NSString *hostNameOverride = _callOptions.hostNameOverride;
if (hostNameOverride) {
args[@GRPC_SSL_TARGET_NAME_OVERRIDE_ARG] = hostNameOverride;
}
if (_callOptions.responseSizeLimit) {
args[@GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH] =
[NSNumber numberWithUnsignedInteger:_callOptions.responseSizeLimit];
}
if (_callOptions.compressionAlgorithm != GRPC_COMPRESS_NONE) {
args[@GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM] =
[NSNumber numberWithInt:_callOptions.compressionAlgorithm];
}
if (_callOptions.keepaliveInterval != 0) {
args[@GRPC_ARG_KEEPALIVE_TIME_MS] =
[NSNumber numberWithUnsignedInteger:(NSUInteger)(_callOptions.keepaliveInterval * 1000)];
args[@GRPC_ARG_KEEPALIVE_TIMEOUT_MS] =
[NSNumber numberWithUnsignedInteger:(NSUInteger)(_callOptions.keepaliveTimeout * 1000)];
}
if (_callOptions.retryEnabled == NO) {
args[@GRPC_ARG_ENABLE_RETRIES] = [NSNumber numberWithInt:_callOptions.retryEnabled];
}
if (_callOptions.connectMinTimeout > 0) {
args[@GRPC_ARG_MIN_RECONNECT_BACKOFF_MS] =
[NSNumber numberWithUnsignedInteger:(NSUInteger)(_callOptions.connectMinTimeout * 1000)];
}
if (_callOptions.connectInitialBackoff > 0) {
args[@GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS] = [NSNumber
numberWithUnsignedInteger:(NSUInteger)(_callOptions.connectInitialBackoff * 1000)];
}
if (_callOptions.connectMaxBackoff > 0) {
args[@GRPC_ARG_MAX_RECONNECT_BACKOFF_MS] =
[NSNumber numberWithUnsignedInteger:(NSUInteger)(_callOptions.connectMaxBackoff * 1000)];
}
if (_callOptions.logContext != nil) {
args[@GRPC_ARG_MOBILE_LOG_CONTEXT] = _callOptions.logContext;
}
if (_callOptions.channelPoolDomain.length != 0) {
args[@GRPC_ARG_CHANNEL_POOL_DOMAIN] = _callOptions.channelPoolDomain;
}
[args addEntriesFromDictionary:_callOptions.additionalChannelArgs];
return args;
}
- (nonnull id)copyWithZone:(nullable NSZone *)zone {
GRPCChannelConfiguration *newConfig =
[[GRPCChannelConfiguration alloc] initWithHost:_host callOptions:_callOptions];
return newConfig;
@implementation GRPCChannelPool {
NSMutableDictionary<GRPCChannelConfiguration *, GRPCChannel *> *_channelPool;
}
- (BOOL)isEqual:(id)object {
if (![object isKindOfClass:[GRPCChannelConfiguration class]]) {
return NO;
+ (nullable instancetype)sharedInstance {
dispatch_once(&gInitChannelPool, ^{
gChannelPool = [[GRPCChannelPool alloc] init];
if (gChannelPool == nil) {
[NSException raise:NSMallocException format:@"Cannot initialize global channel pool."];
}
GRPCChannelConfiguration *obj = (GRPCChannelConfiguration *)object;
if (!(obj.host == _host || (_host != nil && [obj.host isEqualToString:_host]))) return NO;
if (!(obj.callOptions == _callOptions || [obj.callOptions hasChannelOptionsEqualTo:_callOptions]))
return NO;
return YES;
}
- (NSUInteger)hash {
NSUInteger result = 0;
result ^= _host.hash;
result ^= _callOptions.channelOptionsHash;
return result;
}
@end
#pragma mark GRPCChannelPool
@implementation GRPCChannelPool {
NSMutableDictionary<GRPCChannelConfiguration *, GRPCChannel *> *_channelPool;
});
return gChannelPool;
}
- (instancetype)init {
@ -187,61 +63,56 @@ extern const char *kCFStreamVarName;
return self;
}
- (void)dealloc {
[GRPCConnectivityMonitor unregisterObserver:self];
- (GRPCChannel *)channelWithHost:(NSString *)host
callOptions:(GRPCCallOptions *)callOptions {
return [self channelWithHost:host
callOptions:callOptions
destroyDelay:0];
}
- (GRPCChannel *)channelWithConfiguration:(GRPCChannelConfiguration *)configuration {
NSAssert(configuration != nil, @"Must has a configuration");
- (GRPCChannel *)channelWithHost:(NSString *)host
callOptions:(GRPCCallOptions *)callOptions
destroyDelay:(NSTimeInterval)destroyDelay {
NSAssert(host.length > 0, @"Host must not be empty.");
NSAssert(callOptions != nil, @"callOptions must not be empty.");
GRPCChannel *channel;
GRPCChannelConfiguration *configuration =
[[GRPCChannelConfiguration alloc] initWithHost:host callOptions:callOptions];
@synchronized(self) {
if ([_channelPool objectForKey:configuration]) {
channel = _channelPool[configuration];
[channel ref];
if (channel == nil || channel.disconnected) {
if (destroyDelay == 0) {
channel = [[GRPCChannel alloc] initWithChannelConfiguration:configuration];
} else {
channel = [GRPCChannel createChannelWithConfiguration:configuration];
if (channel != nil) {
_channelPool[configuration] = channel;
channel = [[GRPCChannel alloc] initWithChannelConfiguration:configuration destroyDelay:destroyDelay];
}
_channelPool[configuration] = channel;
}
}
return channel;
}
- (void)removeChannel:(GRPCChannel *)channel {
@synchronized(self) {
__block GRPCChannelConfiguration *keyToDelete = nil;
[_channelPool
enumerateKeysAndObjectsUsingBlock:^(GRPCChannelConfiguration *_Nonnull key,
GRPCChannel *_Nonnull obj, BOOL *_Nonnull stop) {
if (obj == channel) {
keyToDelete = key;
*stop = YES;
}
}];
[self->_channelPool removeObjectForKey:keyToDelete];
}
}
- (void)removeAllChannels {
@synchronized(self) {
_channelPool = [NSMutableDictionary dictionary];
}
+ (void)closeOpenConnections {
[[GRPCChannelPool sharedInstance] destroyAllChannels];
}
- (void)removeAndCloseAllChannels {
- (void)destroyAllChannels {
@synchronized(self) {
[_channelPool
enumerateKeysAndObjectsUsingBlock:^(GRPCChannelConfiguration *_Nonnull key,
GRPCChannel *_Nonnull obj, BOOL *_Nonnull stop) {
[obj disconnect];
}];
for (id key in _channelPool) {
[_channelPool[key] disconnect];
}
_channelPool = [NSMutableDictionary dictionary];
}
}
- (void)connectivityChange:(NSNotification *)note {
[self removeAndCloseAllChannels];
[self destroyAllChannels];
}
- (void)dealloc {
[GRPCConnectivityMonitor unregisterObserver:self];
}
@end

@ -24,6 +24,7 @@
#include <grpc/support/alloc.h>
#import "GRPCChannel.h"
#import "GRPCChannelPool.h"
#import "GRPCCompletionQueue.h"
#import "GRPCHost.h"
#import "NSData+GRPC.h"
@ -256,13 +257,21 @@
// consuming too many threads and having contention of multiple calls in a single completion
// queue. Currently we use a singleton queue.
_queue = [GRPCCompletionQueue completionQueue];
_channel = [GRPCChannel channelWithHost:host callOptions:callOptions];
BOOL disconnected;
do {
_channel = [[GRPCChannelPool sharedInstance] channelWithHost:host callOptions:callOptions];
if (_channel == nil) {
NSLog(@"Failed to get a channel for the host.");
return nil;
}
_call = [_channel unmanagedCallWithPath:path completionQueue:_queue callOptions:callOptions];
if (_call == NULL) {
_call = [_channel unmanagedCallWithPath:path
completionQueue:_queue
callOptions:callOptions
disconnected:&disconnected];
// Try create another channel if the current channel is disconnected (due to idleness or
// connectivity monitor disconnection).
} while (_call == NULL && disconnected);
if (_call == nil) {
NSLog(@"Failed to create a call.");
return nil;
}
@ -317,6 +326,7 @@
- (void)dealloc {
if (_call) {
grpc_call_unref(_call);
[_channel unref];
}
[_channel unref];
_channel = nil;

@ -20,6 +20,7 @@
#import "../../GRPCClient/private/GRPCChannel.h"
#import "../../GRPCClient/private/GRPCChannelPool.h"
#import "../../GRPCClient/private/GRPCCompletionQueue.h"
#define TEST_TIMEOUT 32
@ -35,92 +36,104 @@ NSString *kDummyHost = @"dummy.host";
grpc_init();
}
- (void)testCreateChannel {
- (void)testChannelPooling {
NSString *kDummyHost = @"dummy.host";
NSString *kDummyHost2 = @"dummy.host2";
GRPCMutableCallOptions *options1 = [[GRPCMutableCallOptions alloc] init];
options1.transportType = GRPCTransportTypeInsecure;
GRPCCallOptions *options2 = [options1 copy];
GRPCChannelConfiguration *config1 =
[[GRPCChannelConfiguration alloc] initWithHost:kDummyHost callOptions:options1];
GRPCChannelConfiguration *config2 =
[[GRPCChannelConfiguration alloc] initWithHost:kDummyHost callOptions:options2];
GRPCChannelPool *pool = [[GRPCChannelPool alloc] init];
GRPCChannel *channel1 = [pool channelWithConfiguration:config1];
GRPCChannel *channel2 = [pool channelWithConfiguration:config2];
GRPCMutableCallOptions *options3 = [options2 mutableCopy];
options3.transportType = GRPCTransportTypeInsecure;
GRPCChannelPool *pool = [GRPCChannelPool sharedInstance];
GRPCChannel *channel1 = [pool channelWithHost:kDummyHost
callOptions:options1];
GRPCChannel *channel2 = [pool channelWithHost:kDummyHost
callOptions:options2];
GRPCChannel *channel3 = [pool channelWithHost:kDummyHost2
callOptions:options1];
GRPCChannel *channel4 = [pool channelWithHost:kDummyHost
callOptions:options3];
XCTAssertEqual(channel1, channel2);
XCTAssertNotEqual(channel1, channel3);
XCTAssertNotEqual(channel1, channel4);
XCTAssertNotEqual(channel3, channel4);
}
- (void)testChannelRemove {
GRPCMutableCallOptions *options1 = [[GRPCMutableCallOptions alloc] init];
options1.transportType = GRPCTransportTypeInsecure;
GRPCChannelConfiguration *config1 =
[[GRPCChannelConfiguration alloc] initWithHost:kDummyHost callOptions:options1];
GRPCChannelPool *pool = [[GRPCChannelPool alloc] init];
GRPCChannel *channel1 = [pool channelWithConfiguration:config1];
[pool removeChannel:channel1];
GRPCChannel *channel2 = [pool channelWithConfiguration:config1];
XCTAssertNotEqual(channel1, channel2);
}
extern NSTimeInterval kChannelDestroyDelay;
- (void)testDestroyAllChannels {
NSString *kDummyHost = @"dummy.host";
- (void)testChannelTimeoutCancel {
NSTimeInterval kOriginalInterval = kChannelDestroyDelay;
kChannelDestroyDelay = 3.0;
GRPCMutableCallOptions *options1 = [[GRPCMutableCallOptions alloc] init];
options1.transportType = GRPCTransportTypeInsecure;
GRPCChannelConfiguration *config1 =
[[GRPCChannelConfiguration alloc] initWithHost:kDummyHost callOptions:options1];
GRPCChannelPool *pool = [[GRPCChannelPool alloc] init];
GRPCChannel *channel1 = [pool channelWithConfiguration:config1];
[channel1 unref];
sleep(1);
GRPCChannel *channel2 = [pool channelWithConfiguration:config1];
XCTAssertEqual(channel1, channel2);
sleep((int)kChannelDestroyDelay + 2);
GRPCChannel *channel3 = [pool channelWithConfiguration:config1];
XCTAssertEqual(channel1, channel3);
kChannelDestroyDelay = kOriginalInterval;
GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
GRPCChannelPool *pool = [GRPCChannelPool sharedInstance];
GRPCChannel *channel = [pool channelWithHost:kDummyHost
callOptions:options];
grpc_call *call = [channel unmanagedCallWithPath:@"dummy.path"
completionQueue:[GRPCCompletionQueue completionQueue]
callOptions:options
disconnected:nil];
[pool destroyAllChannels];
XCTAssertTrue(channel.disconnected);
GRPCChannel *channel2 = [pool channelWithHost:kDummyHost
callOptions:options];
XCTAssertNotEqual(channel, channel2);
grpc_call_unref(call);
}
- (void)testChannelDisconnect {
- (void)testGetChannelBeforeChannelTimedDisconnection {
NSString *kDummyHost = @"dummy.host";
GRPCMutableCallOptions *options1 = [[GRPCMutableCallOptions alloc] init];
options1.transportType = GRPCTransportTypeInsecure;
GRPCCallOptions *options2 = [options1 copy];
GRPCChannelConfiguration *config1 =
[[GRPCChannelConfiguration alloc] initWithHost:kDummyHost callOptions:options1];
GRPCChannelConfiguration *config2 =
[[GRPCChannelConfiguration alloc] initWithHost:kDummyHost callOptions:options2];
GRPCChannelPool *pool = [[GRPCChannelPool alloc] init];
GRPCChannel *channel1 = [pool channelWithConfiguration:config1];
[pool removeAndCloseAllChannels];
GRPCChannel *channel2 = [pool channelWithConfiguration:config2];
XCTAssertNotEqual(channel1, channel2);
const NSTimeInterval kDestroyDelay = 1;
GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
GRPCChannelPool *pool = [GRPCChannelPool sharedInstance];
GRPCChannel *channel = [pool channelWithHost:kDummyHost
callOptions:options
destroyDelay:kDestroyDelay];
grpc_call *call = [channel unmanagedCallWithPath:@"dummy.path"
completionQueue:[GRPCCompletionQueue completionQueue]
callOptions:options
disconnected:nil];
grpc_call_unref(call);
[channel unref];
// Test that we can still get the channel at this time
GRPCChannel *channel2 = [pool channelWithHost:kDummyHost
callOptions:options
destroyDelay:kDestroyDelay];
XCTAssertEqual(channel, channel2);
call = [channel2 unmanagedCallWithPath:@"dummy.path"
completionQueue:[GRPCCompletionQueue completionQueue]
callOptions:options
disconnected:nil];
// Test that after the destroy delay, the channel is still alive
sleep(kDestroyDelay + 1);
XCTAssertFalse(channel.disconnected);
}
- (void)testClearChannels {
GRPCMutableCallOptions *options1 = [[GRPCMutableCallOptions alloc] init];
options1.transportType = GRPCTransportTypeInsecure;
GRPCMutableCallOptions *options2 = [[GRPCMutableCallOptions alloc] init];
options2.transportType = GRPCTransportTypeChttp2BoringSSL;
GRPCChannelConfiguration *config1 =
[[GRPCChannelConfiguration alloc] initWithHost:kDummyHost callOptions:options1];
GRPCChannelConfiguration *config2 =
[[GRPCChannelConfiguration alloc] initWithHost:kDummyHost callOptions:options2];
GRPCChannelPool *pool = [[GRPCChannelPool alloc] init];
GRPCChannel *channel1 = [pool channelWithConfiguration:config1];
GRPCChannel *channel2 = [pool channelWithConfiguration:config2];
XCTAssertNotEqual(channel1, channel2);
[pool removeAndCloseAllChannels];
GRPCChannel *channel3 = [pool channelWithConfiguration:config1];
GRPCChannel *channel4 = [pool channelWithConfiguration:config2];
XCTAssertNotEqual(channel1, channel3);
XCTAssertNotEqual(channel2, channel4);
- (void)testGetChannelAfterChannelTimedDisconnection {
NSString *kDummyHost = @"dummy.host";
const NSTimeInterval kDestroyDelay = 1;
GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
GRPCChannelPool *pool = [GRPCChannelPool sharedInstance];
GRPCChannel *channel = [pool channelWithHost:kDummyHost
callOptions:options
destroyDelay:kDestroyDelay];
grpc_call *call = [channel unmanagedCallWithPath:@"dummy.path"
completionQueue:[GRPCCompletionQueue completionQueue]
callOptions:options
disconnected:nil];
grpc_call_unref(call);
[channel unref];
sleep(kDestroyDelay + 1);
// Test that we get new channel to the same host and with the same callOptions
GRPCChannel *channel2 = [pool channelWithHost:kDummyHost
callOptions:options
destroyDelay:kDestroyDelay];
XCTAssertNotEqual(channel, channel2);
}
@end

@ -20,6 +20,7 @@
#import "../../GRPCClient/GRPCCallOptions.h"
#import "../../GRPCClient/private/GRPCChannel.h"
#import "../../GRPCClient/private/GRPCCompletionQueue.h"
@interface ChannelTests : XCTestCase
@ -31,63 +32,51 @@
grpc_init();
}
- (void)testSameConfiguration {
NSString *host = @"grpc-test.sandbox.googleapis.com";
GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
options.userAgentPrefix = @"TestUAPrefix";
NSMutableDictionary *args = [NSMutableDictionary new];
args[@"abc"] = @"xyz";
options.additionalChannelArgs = [args copy];
GRPCChannel *channel1 = [GRPCChannel channelWithHost:host callOptions:options];
GRPCChannel *channel2 = [GRPCChannel channelWithHost:host callOptions:options];
XCTAssertEqual(channel1, channel2);
GRPCMutableCallOptions *options2 = [options mutableCopy];
options2.additionalChannelArgs = [args copy];
GRPCChannel *channel3 = [GRPCChannel channelWithHost:host callOptions:options2];
XCTAssertEqual(channel1, channel3);
}
- (void)testTimedDisconnection {
NSString * const kHost = @"grpc-test.sandbox.googleapis.com";
const NSTimeInterval kDestroyDelay = 1;
GRPCCallOptions *options = [[GRPCCallOptions alloc] init];
GRPCChannelConfiguration *configuration = [[GRPCChannelConfiguration alloc] initWithHost:kHost callOptions:options];
GRPCChannel *channel = [[GRPCChannel alloc] initWithChannelConfiguration:configuration
destroyDelay:kDestroyDelay];
BOOL disconnected;
grpc_call *call = [channel unmanagedCallWithPath:@"dummy.path"
completionQueue:[GRPCCompletionQueue completionQueue]
callOptions:options
disconnected:&disconnected];
XCTAssertFalse(disconnected);
grpc_call_unref(call);
[channel unref];
XCTAssertFalse(channel.disconnected, @"Channel is pre-maturely disconnected.");
sleep(kDestroyDelay + 1);
XCTAssertTrue(channel.disconnected, @"Channel is not disconnected after delay.");
- (void)testDifferentHost {
NSString *host1 = @"grpc-test.sandbox.googleapis.com";
NSString *host2 = @"grpc-test2.sandbox.googleapis.com";
NSString *host3 = @"http://grpc-test.sandbox.googleapis.com";
NSString *host4 = @"dns://grpc-test.sandbox.googleapis.com";
NSString *host5 = @"grpc-test.sandbox.googleapis.com:80";
GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
options.userAgentPrefix = @"TestUAPrefix";
NSMutableDictionary *args = [NSMutableDictionary new];
args[@"abc"] = @"xyz";
options.additionalChannelArgs = [args copy];
GRPCChannel *channel1 = [GRPCChannel channelWithHost:host1 callOptions:options];
GRPCChannel *channel2 = [GRPCChannel channelWithHost:host2 callOptions:options];
GRPCChannel *channel3 = [GRPCChannel channelWithHost:host3 callOptions:options];
GRPCChannel *channel4 = [GRPCChannel channelWithHost:host4 callOptions:options];
GRPCChannel *channel5 = [GRPCChannel channelWithHost:host5 callOptions:options];
XCTAssertNotEqual(channel1, channel2);
XCTAssertNotEqual(channel1, channel3);
XCTAssertNotEqual(channel1, channel4);
XCTAssertNotEqual(channel1, channel5);
// Check another call creation returns null and indicates disconnected.
call = [channel unmanagedCallWithPath:@"dummy.path"
completionQueue:[GRPCCompletionQueue completionQueue]
callOptions:options
disconnected:&disconnected];
XCTAssert(call == NULL);
XCTAssertTrue(disconnected);
}
- (void)testDifferentChannelParameters {
NSString *host = @"grpc-test.sandbox.googleapis.com";
GRPCMutableCallOptions *options1 = [[GRPCMutableCallOptions alloc] init];
options1.transportType = GRPCTransportTypeChttp2BoringSSL;
NSMutableDictionary *args = [NSMutableDictionary new];
args[@"abc"] = @"xyz";
options1.additionalChannelArgs = [args copy];
GRPCMutableCallOptions *options2 = [[GRPCMutableCallOptions alloc] init];
options2.transportType = GRPCTransportTypeInsecure;
options2.additionalChannelArgs = [args copy];
GRPCMutableCallOptions *options3 = [[GRPCMutableCallOptions alloc] init];
options3.transportType = GRPCTransportTypeChttp2BoringSSL;
args[@"def"] = @"uvw";
options3.additionalChannelArgs = [args copy];
GRPCChannel *channel1 = [GRPCChannel channelWithHost:host callOptions:options1];
GRPCChannel *channel2 = [GRPCChannel channelWithHost:host callOptions:options2];
GRPCChannel *channel3 = [GRPCChannel channelWithHost:host callOptions:options3];
XCTAssertNotEqual(channel1, channel2);
XCTAssertNotEqual(channel1, channel3);
- (void)testForceDisconnection {
NSString * const kHost = @"grpc-test.sandbox.googleapis.com";
const NSTimeInterval kDestroyDelay = 1;
GRPCCallOptions *options = [[GRPCCallOptions alloc] init];
GRPCChannelConfiguration *configuration = [[GRPCChannelConfiguration alloc] initWithHost:kHost callOptions:options];
GRPCChannel *channel = [[GRPCChannel alloc] initWithChannelConfiguration:configuration
destroyDelay:kDestroyDelay];
grpc_call *call = [channel unmanagedCallWithPath:@"dummy.path"
completionQueue:[GRPCCompletionQueue completionQueue]
callOptions:options
disconnected:nil];
grpc_call_unref(call);
[channel disconnect];
XCTAssertTrue(channel.disconnected, @"Channel is not disconnected.");
// Test calling another unref here will not crash
[channel unref];
}
@end

Loading…
Cancel
Save