From 7d55a8a904010cdb2b03fb1edddf8522ad5e9f69 Mon Sep 17 00:00:00 2001 From: Yusuke Sato Date: Tue, 30 Apr 2019 15:24:50 +0900 Subject: [PATCH 01/55] Fix a typo in the GRPC::RpcServer constructor description --- src/ruby/lib/grpc/generic/rpc_server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb index 3d34419643e..a566afa0bc2 100644 --- a/src/ruby/lib/grpc/generic/rpc_server.rb +++ b/src/ruby/lib/grpc/generic/rpc_server.rb @@ -210,7 +210,7 @@ module GRPC # A server arguments hash to be passed down to the underlying core server # # * interceptors: - # Am array of GRPC::ServerInterceptor objects that will be used for + # An array of GRPC::ServerInterceptor objects that will be used for # intercepting server handlers to provide extra functionality. # Interceptors are an EXPERIMENTAL API. # From 93c8088b332c02e505743e3ad06ecbd31ccb863a Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 10 Jun 2019 12:10:08 -0700 Subject: [PATCH 02/55] Implement global interceptor --- .../GRPCClient/GRPCCall+Interceptor.h | 29 ++++ .../GRPCClient/GRPCCall+Interceptor.m | 60 +++++++ src/objective-c/GRPCClient/GRPCCall.m | 37 +++- .../tests/InteropTests/InteropTests.m | 158 +++++++++++++++++- 4 files changed, 277 insertions(+), 7 deletions(-) create mode 100644 src/objective-c/GRPCClient/GRPCCall+Interceptor.h create mode 100644 src/objective-c/GRPCClient/GRPCCall+Interceptor.m diff --git a/src/objective-c/GRPCClient/GRPCCall+Interceptor.h b/src/objective-c/GRPCClient/GRPCCall+Interceptor.h new file mode 100644 index 00000000000..953846d8c99 --- /dev/null +++ b/src/objective-c/GRPCClient/GRPCCall+Interceptor.h @@ -0,0 +1,29 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#import "GRPCCall.h" + +@protocol GRPCInterceptorFactory; + +@interface GRPCCall2 (Interceptor) + ++ (void)registerGlobalInterceptor:(id)interceptorFactory; + ++ (id)globalInterceptorFactory; + +@end diff --git a/src/objective-c/GRPCClient/GRPCCall+Interceptor.m b/src/objective-c/GRPCClient/GRPCCall+Interceptor.m new file mode 100644 index 00000000000..b1cad75d4f3 --- /dev/null +++ b/src/objective-c/GRPCClient/GRPCCall+Interceptor.m @@ -0,0 +1,60 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#import "GRPCCall+Interceptor.h" +#import "GRPCInterceptor.h" + +static id globalInterceptorFactory = nil; +static NSLock *globalInterceptorLock = nil; +static dispatch_once_t onceToken; + +@implementation GRPCCall2 (Interceptor) + ++ (void)registerGlobalInterceptor:(id)interceptorFactory { + dispatch_once(&onceToken, ^{ + globalInterceptorLock = [[NSLock alloc] init]; + }); + [globalInterceptorLock lock]; + NSAssert(globalInterceptorFactory == nil, + @"Global interceptor is already registered. Only one global interceptor can be " + @"registered in a process"); + if (globalInterceptorFactory != nil) { + NSLog( + @"Global interceptor is already registered. Only one global interceptor can be registered " + @"in a process"); + [globalInterceptorLock unlock]; + return; + } + + globalInterceptorFactory = interceptorFactory; + [globalInterceptorLock unlock]; +} + ++ (id)globalInterceptorFactory { + dispatch_once(&onceToken, ^{ + globalInterceptorLock = [[NSLock alloc] init]; + }); + id factory; + [globalInterceptorLock lock]; + factory = globalInterceptorFactory; + [globalInterceptorLock unlock]; + + return factory; +} + +@end diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index e97ed6e3a9e..684867ae1ee 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -17,6 +17,7 @@ */ #import "GRPCCall.h" +#import "GRPCCall+Interceptor.h" #import "GRPCCall+OAuth2.h" #import "GRPCCallOptions.h" #import "GRPCInterceptor.h" @@ -141,23 +142,51 @@ const char *kCFStreamVarName = "grpc_cfstream"; _responseHandler = responseHandler; // Initialize the interceptor chain + + // First initialize the internal call GRPCCall2Internal *internalCall = [[GRPCCall2Internal alloc] init]; id nextInterceptor = internalCall; GRPCInterceptorManager *nextManager = nil; + + // Then initialize the global interceptor, if applicable + id globalInterceptorFactory = [GRPCCall2 globalInterceptorFactory]; + if (globalInterceptorFactory) { + GRPCInterceptorManager *manager = + [[GRPCInterceptorManager alloc] initWithNextInterceptor:nextInterceptor]; + GRPCInterceptor *interceptor = + [globalInterceptorFactory createInterceptorWithManager:manager]; + NSAssert(interceptor != nil, @"Failed to create interceptor from factory: %@", + globalInterceptorFactory); + if (interceptor == nil) { + NSLog(@"Failed to create interceptor from factory: %@", globalInterceptorFactory); + } else { + [internalCall setResponseHandler:interceptor]; + } + nextInterceptor = interceptor; + nextManager = manager; + } + + // Finanlly initialize the interceptors in the chain NSArray *interceptorFactories = _actualCallOptions.interceptorFactories; if (interceptorFactories.count == 0) { - [internalCall setResponseHandler:_responseHandler]; + if (nextManager == nil) { + [internalCall setResponseHandler:_responseHandler]; + } else { + [nextManager setPreviousInterceptor:_responseHandler]; + } } else { for (int i = (int)interceptorFactories.count - 1; i >= 0; i--) { GRPCInterceptorManager *manager = [[GRPCInterceptorManager alloc] initWithNextInterceptor:nextInterceptor]; GRPCInterceptor *interceptor = [interceptorFactories[i] createInterceptorWithManager:manager]; - NSAssert(interceptor != nil, @"Failed to create interceptor"); + NSAssert(interceptor != nil, @"Failed to create interceptor from factory: %@", + interceptorFactories[i]); if (interceptor == nil) { - return nil; + NSLog(@"Failed to create interceptor from factory: %@", interceptorFactories[i]); + continue; } - if (i == (int)interceptorFactories.count - 1) { + if (nextManager == nil) { [internalCall setResponseHandler:interceptor]; } else { [nextManager setPreviousInterceptor:interceptor]; diff --git a/src/objective-c/tests/InteropTests/InteropTests.m b/src/objective-c/tests/InteropTests/InteropTests.m index 91e133a1300..ef22a53cd96 100644 --- a/src/objective-c/tests/InteropTests/InteropTests.m +++ b/src/objective-c/tests/InteropTests/InteropTests.m @@ -25,6 +25,7 @@ #endif #import #import +#import #import #import #import @@ -1224,7 +1225,8 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager - (void)testDefaultInterceptor { XCTAssertNotNil([[self class] host]); - __weak XCTestExpectation *expectation = [self expectationWithDescription:@"PingPongWithV2API"]; + __weak XCTestExpectation *expectation = + [self expectationWithDescription:@"testDefaultInterceptor"]; NSArray *requests = @[ @27182, @8, @1828, @45904 ]; NSArray *responses = @[ @31415, @9, @2653, @58979 ]; @@ -1277,7 +1279,8 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager - (void)testLoggingInterceptor { XCTAssertNotNil([[self class] host]); - __weak XCTestExpectation *expectation = [self expectationWithDescription:@"PingPongWithV2API"]; + __weak XCTestExpectation *expectation = + [self expectationWithDescription:@"testLoggingInterceptor"]; __block NSUInteger startCount = 0; __block NSUInteger writeDataCount = 0; @@ -1400,7 +1403,8 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager - (void)testHijackingInterceptor { NSUInteger kCancelAfterWrites = 2; XCTAssertNotNil([[self class] host]); - __weak XCTestExpectation *expectation = [self expectationWithDescription:@"PingPongWithV2API"]; + __weak XCTestExpectation *expectation = + [self expectationWithDescription:@"testHijackingInterceptor"]; NSArray *responses = @[ @1, @2, @3, @4 ]; __block int index = 0; @@ -1514,4 +1518,152 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager XCTAssertEqual(responseCloseCount, 1); } +- (void)testGlobalInterceptor { + XCTAssertNotNil([[self class] host]); + __weak XCTestExpectation *expectation = + [self expectationWithDescription:@"testLoggingInterceptor"]; + + __block NSUInteger startCount = 0; + __block NSUInteger writeDataCount = 0; + __block NSUInteger finishCount = 0; + __block NSUInteger receiveNextMessageCount = 0; + __block NSUInteger responseHeaderCount = 0; + __block NSUInteger responseDataCount = 0; + __block NSUInteger responseCloseCount = 0; + __block NSUInteger didWriteDataCount = 0; + id factory = [[HookInterceptorFactory alloc] + initWithRequestDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + responseDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + startHook:^(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, + GRPCInterceptorManager *manager) { + startCount++; + XCTAssertEqualObjects(requestOptions.host, [[self class] host]); + XCTAssertEqualObjects(requestOptions.path, @"/grpc.testing.TestService/FullDuplexCall"); + XCTAssertEqual(requestOptions.safety, GRPCCallSafetyDefault); + [manager startNextInterceptorWithRequest:[requestOptions copy] + callOptions:[callOptions copy]]; + } + writeDataHook:^(id data, GRPCInterceptorManager *manager) { + writeDataCount++; + [manager writeNextInterceptorWithData:data]; + } + finishHook:^(GRPCInterceptorManager *manager) { + finishCount++; + [manager finishNextInterceptor]; + } + receiveNextMessagesHook:^(NSUInteger numberOfMessages, GRPCInterceptorManager *manager) { + receiveNextMessageCount++; + [manager receiveNextInterceptorMessages:numberOfMessages]; + } + responseHeaderHook:^(NSDictionary *initialMetadata, GRPCInterceptorManager *manager) { + responseHeaderCount++; + [manager forwardPreviousInterceptorWithInitialMetadata:initialMetadata]; + } + responseDataHook:^(id data, GRPCInterceptorManager *manager) { + responseDataCount++; + [manager forwardPreviousInterceptorWithData:data]; + } + responseCloseHook:^(NSDictionary *trailingMetadata, NSError *error, + GRPCInterceptorManager *manager) { + responseCloseCount++; + [manager forwardPreviousInterceptorCloseWithTrailingMetadata:trailingMetadata error:error]; + } + didWriteDataHook:^(GRPCInterceptorManager *manager) { + didWriteDataCount++; + [manager forwardPreviousInterceptorDidWriteData]; + }]; + + NSArray *requests = @[ @1, @2, @3, @4 ]; + NSArray *responses = @[ @1, @2, @3, @4 ]; + + __block int index = 0; + + id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:requests[index] + requestedResponseSize:responses[index]]; + GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init]; + options.transportType = [[self class] transportType]; + options.PEMRootCertificates = [[self class] PEMRootCertificates]; + options.hostNameOverride = [[self class] hostNameOverride]; + options.flowControlEnabled = YES; + [GRPCCall2 registerGlobalInterceptor:factory]; + + __block BOOL canWriteData = NO; + __block GRPCStreamingProtoCall *call = [_service + fullDuplexCallWithResponseHandler:[[InteropTestsBlockCallbacks alloc] + initWithInitialMetadataCallback:nil + messageCallback:^(id message) { + XCTAssertLessThan(index, 4, + @"More than 4 responses received."); + index += 1; + if (index < 4) { + id request = [RMTStreamingOutputCallRequest + messageWithPayloadSize:requests[index] + requestedResponseSize:responses[index]]; + XCTAssertTrue(canWriteData); + canWriteData = NO; + [call writeMessage:request]; + [call receiveNextMessage]; + } else { + [call finish]; + } + } + closeCallback:^(NSDictionary *trailingMetadata, + NSError *error) { + XCTAssertNil(error, + @"Finished with unexpected error: %@", + error); + [expectation fulfill]; + } + writeMessageCallback:^{ + canWriteData = YES; + }] + callOptions:options]; + [call start]; + [call receiveNextMessage]; + [call writeMessage:request]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; + XCTAssertEqual(startCount, 1); + XCTAssertEqual(writeDataCount, 4); + XCTAssertEqual(finishCount, 1); + XCTAssertEqual(receiveNextMessageCount, 4); + XCTAssertEqual(responseHeaderCount, 1); + XCTAssertEqual(responseDataCount, 4); + XCTAssertEqual(responseCloseCount, 1); + XCTAssertEqual(didWriteDataCount, 4); +} + +- (void)testConflictingGlobalInterceptors { + id factory = [[HookInterceptorFactory alloc] + initWithRequestDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + responseDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + startHook:nil + writeDataHook:nil + finishHook:nil + receiveNextMessagesHook:nil + responseHeaderHook:nil + responseDataHook:nil + responseCloseHook:nil + didWriteDataHook:nil]; + id factory2 = [[HookInterceptorFactory alloc] + initWithRequestDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + responseDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + startHook:nil + writeDataHook:nil + finishHook:nil + receiveNextMessagesHook:nil + responseHeaderHook:nil + responseDataHook:nil + responseCloseHook:nil + didWriteDataHook:nil]; + + [GRPCCall2 registerGlobalInterceptor:factory]; + @try { + [GRPCCall2 registerGlobalInterceptor:factory2]; + XCTFail(@"Did not receive an exception when registering global interceptor the second time"); + } @catch (NSException *exception) { + NSLog(@"Received exception as expected: %@", exception); + } +} + @end From 94a18020eb6652976e73d386f355cf0b08c28fc3 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 13 Jun 2019 16:11:17 -0700 Subject: [PATCH 03/55] nit fixes --- src/objective-c/GRPCClient/GRPCCall.m | 6 +++--- src/objective-c/tests/InteropTests/InteropTests.m | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 684867ae1ee..5280f8d01f4 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -161,12 +161,12 @@ const char *kCFStreamVarName = "grpc_cfstream"; NSLog(@"Failed to create interceptor from factory: %@", globalInterceptorFactory); } else { [internalCall setResponseHandler:interceptor]; + nextInterceptor = interceptor; + nextManager = manager; } - nextInterceptor = interceptor; - nextManager = manager; } - // Finanlly initialize the interceptors in the chain + // Finally initialize the interceptors in the chain NSArray *interceptorFactories = _actualCallOptions.interceptorFactories; if (interceptorFactories.count == 0) { if (nextManager == nil) { diff --git a/src/objective-c/tests/InteropTests/InteropTests.m b/src/objective-c/tests/InteropTests/InteropTests.m index ef22a53cd96..6f00176b20c 100644 --- a/src/objective-c/tests/InteropTests/InteropTests.m +++ b/src/objective-c/tests/InteropTests/InteropTests.m @@ -1521,7 +1521,7 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager - (void)testGlobalInterceptor { XCTAssertNotNil([[self class] host]); __weak XCTestExpectation *expectation = - [self expectationWithDescription:@"testLoggingInterceptor"]; + [self expectationWithDescription:@"testGlobalInterceptor"]; __block NSUInteger startCount = 0; __block NSUInteger writeDataCount = 0; From f08aab26bf132258a75e04266fd217196c5033d6 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 13 Jun 2019 16:11:51 -0700 Subject: [PATCH 04/55] Add test for 1 global interceptor and 1 local interceptor --- .../tests/InteropTests/InteropTests.m | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/src/objective-c/tests/InteropTests/InteropTests.m b/src/objective-c/tests/InteropTests/InteropTests.m index 6f00176b20c..962a15cbf8c 100644 --- a/src/objective-c/tests/InteropTests/InteropTests.m +++ b/src/objective-c/tests/InteropTests/InteropTests.m @@ -1666,4 +1666,174 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager } } +- (void)testIntecepptorAndGlobalInterceptor { + XCTAssertNotNil([[self class] host]); + __weak XCTestExpectation *expectation = + [self expectationWithDescription:@"testGlobalInterceptor"]; + + __block NSUInteger startCount = 0; + __block NSUInteger writeDataCount = 0; + __block NSUInteger finishCount = 0; + __block NSUInteger receiveNextMessageCount = 0; + __block NSUInteger responseHeaderCount = 0; + __block NSUInteger responseDataCount = 0; + __block NSUInteger responseCloseCount = 0; + __block NSUInteger didWriteDataCount = 0; + + id factory = [[HookInterceptorFactory alloc] + initWithRequestDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + responseDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + startHook:^(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, + GRPCInterceptorManager *manager) { + startCount++; + XCTAssertEqualObjects(requestOptions.host, [[self class] host]); + XCTAssertEqualObjects(requestOptions.path, @"/grpc.testing.TestService/FullDuplexCall"); + XCTAssertEqual(requestOptions.safety, GRPCCallSafetyDefault); + [manager startNextInterceptorWithRequest:[requestOptions copy] + callOptions:[callOptions copy]]; + } + writeDataHook:^(id data, GRPCInterceptorManager *manager) { + writeDataCount++; + [manager writeNextInterceptorWithData:data]; + } + finishHook:^(GRPCInterceptorManager *manager) { + finishCount++; + [manager finishNextInterceptor]; + } + receiveNextMessagesHook:^(NSUInteger numberOfMessages, GRPCInterceptorManager *manager) { + receiveNextMessageCount++; + [manager receiveNextInterceptorMessages:numberOfMessages]; + } + responseHeaderHook:^(NSDictionary *initialMetadata, GRPCInterceptorManager *manager) { + responseHeaderCount++; + [manager forwardPreviousInterceptorWithInitialMetadata:initialMetadata]; + } + responseDataHook:^(id data, GRPCInterceptorManager *manager) { + responseDataCount++; + [manager forwardPreviousInterceptorWithData:data]; + } + responseCloseHook:^(NSDictionary *trailingMetadata, NSError *error, + GRPCInterceptorManager *manager) { + responseCloseCount++; + [manager forwardPreviousInterceptorCloseWithTrailingMetadata:trailingMetadata error:error]; + } + didWriteDataHook:^(GRPCInterceptorManager *manager) { + didWriteDataCount++; + [manager forwardPreviousInterceptorDidWriteData]; + }]; + + __block NSUInteger globalStartCount = 0; + __block NSUInteger globalWriteDataCount = 0; + __block NSUInteger globalFinishCount = 0; + __block NSUInteger globalReceiveNextMessageCount = 0; + __block NSUInteger globalResponseHeaderCount = 0; + __block NSUInteger globalResponseDataCount = 0; + __block NSUInteger globalResponseCloseCount = 0; + __block NSUInteger globalDidWriteDataCount = 0; + + id globalFactory = [[HookInterceptorFactory alloc] + initWithRequestDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + responseDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) + startHook:^(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, + GRPCInterceptorManager *manager) { + globalStartCount++; + XCTAssertEqualObjects(requestOptions.host, [[self class] host]); + XCTAssertEqualObjects(requestOptions.path, @"/grpc.testing.TestService/FullDuplexCall"); + XCTAssertEqual(requestOptions.safety, GRPCCallSafetyDefault); + [manager startNextInterceptorWithRequest:[requestOptions copy] + callOptions:[callOptions copy]]; + } + writeDataHook:^(id data, GRPCInterceptorManager *manager) { + globalWriteDataCount++; + [manager writeNextInterceptorWithData:data]; + } + finishHook:^(GRPCInterceptorManager *manager) { + globalFinishCount++; + [manager finishNextInterceptor]; + } + receiveNextMessagesHook:^(NSUInteger numberOfMessages, GRPCInterceptorManager *manager) { + globalReceiveNextMessageCount++; + [manager receiveNextInterceptorMessages:numberOfMessages]; + } + responseHeaderHook:^(NSDictionary *initialMetadata, GRPCInterceptorManager *manager) { + globalResponseHeaderCount++; + [manager forwardPreviousInterceptorWithInitialMetadata:initialMetadata]; + } + responseDataHook:^(id data, GRPCInterceptorManager *manager) { + globalResponseDataCount++; + [manager forwardPreviousInterceptorWithData:data]; + } + responseCloseHook:^(NSDictionary *trailingMetadata, NSError *error, + GRPCInterceptorManager *manager) { + globalResponseCloseCount++; + [manager forwardPreviousInterceptorCloseWithTrailingMetadata:trailingMetadata error:error]; + } + didWriteDataHook:^(GRPCInterceptorManager *manager) { + globalDidWriteDataCount++; + [manager forwardPreviousInterceptorDidWriteData]; + }]; + + NSArray *requests = @[ @1, @2, @3, @4 ]; + NSArray *responses = @[ @1, @2, @3, @4 ]; + + __block int index = 0; + + id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:requests[index] + requestedResponseSize:responses[index]]; + GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init]; + options.transportType = [[self class] transportType]; + options.PEMRootCertificates = [[self class] PEMRootCertificates]; + options.hostNameOverride = [[self class] hostNameOverride]; + options.flowControlEnabled = YES; + options.interceptorFactories = @[ factory ]; + [GRPCCall2 registerGlobalInterceptor:globalFactory]; + + __block BOOL canWriteData = NO; + __block GRPCStreamingProtoCall *call = [_service + fullDuplexCallWithResponseHandler:[[InteropTestsBlockCallbacks alloc] + initWithInitialMetadataCallback:nil + messageCallback:^(id message) { + index += 1; + if (index < 4) { + id request = [RMTStreamingOutputCallRequest + messageWithPayloadSize:requests[index] + requestedResponseSize:responses[index]]; + canWriteData = NO; + [call writeMessage:request]; + [call receiveNextMessage]; + } else { + [call finish]; + } + } + closeCallback:^(NSDictionary *trailingMetadata, + NSError *error) { + [expectation fulfill]; + } + writeMessageCallback:^{ + canWriteData = YES; + }] + callOptions:options]; + [call start]; + [call receiveNextMessage]; + [call writeMessage:request]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; + XCTAssertEqual(startCount, 1); + XCTAssertEqual(writeDataCount, 4); + XCTAssertEqual(finishCount, 1); + XCTAssertEqual(receiveNextMessageCount, 4); + XCTAssertEqual(responseHeaderCount, 1); + XCTAssertEqual(responseDataCount, 4); + XCTAssertEqual(responseCloseCount, 1); + XCTAssertEqual(didWriteDataCount, 4); + XCTAssertEqual(globalStartCount, 1); + XCTAssertEqual(globalWriteDataCount, 4); + XCTAssertEqual(globalFinishCount, 1); + XCTAssertEqual(globalReceiveNextMessageCount, 4); + XCTAssertEqual(globalResponseHeaderCount, 1); + XCTAssertEqual(globalResponseDataCount, 4); + XCTAssertEqual(globalResponseCloseCount, 1); + XCTAssertEqual(globalDidWriteDataCount, 4); +} + @end From 19c9b193c560895292917415aac7ffe5333229f1 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 14 Jun 2019 08:11:39 -0700 Subject: [PATCH 05/55] fix typos --- src/objective-c/tests/InteropTests/InteropTests.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/tests/InteropTests/InteropTests.m b/src/objective-c/tests/InteropTests/InteropTests.m index 962a15cbf8c..2c54fb80f3d 100644 --- a/src/objective-c/tests/InteropTests/InteropTests.m +++ b/src/objective-c/tests/InteropTests/InteropTests.m @@ -1666,10 +1666,10 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager } } -- (void)testIntecepptorAndGlobalInterceptor { +- (void)testInterceptorAndGlobalInterceptor { XCTAssertNotNil([[self class] host]); __weak XCTestExpectation *expectation = - [self expectationWithDescription:@"testGlobalInterceptor"]; + [self expectationWithDescription:@"testInterceptorAndGlobalInterceptor"]; __block NSUInteger startCount = 0; __block NSUInteger writeDataCount = 0; From 5b329fd1aa5a6a1ff4ae1d9b6c83e33342392c31 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 14 Jun 2019 13:17:37 -0700 Subject: [PATCH 06/55] Add experimental notice to global interceptor --- src/objective-c/GRPCClient/GRPCCall+Interceptor.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/objective-c/GRPCClient/GRPCCall+Interceptor.h b/src/objective-c/GRPCClient/GRPCCall+Interceptor.h index 953846d8c99..4183e0aa20e 100644 --- a/src/objective-c/GRPCClient/GRPCCall+Interceptor.h +++ b/src/objective-c/GRPCClient/GRPCCall+Interceptor.h @@ -16,6 +16,8 @@ * */ +// The global interceptor feature is experimental and might be modified or removed at any time. + #import "GRPCCall.h" @protocol GRPCInterceptorFactory; From e5de1e5e254c00c912032eef86399e2c911771fc Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 18 Jun 2019 10:12:41 -0700 Subject: [PATCH 07/55] Update global interceptor comment and raise exception --- src/objective-c/GRPCClient/GRPCCall+Interceptor.h | 11 +++++++++-- src/objective-c/GRPCClient/GRPCCall+Interceptor.m | 10 ++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall+Interceptor.h b/src/objective-c/GRPCClient/GRPCCall+Interceptor.h index 4183e0aa20e..e6da2a7bda4 100644 --- a/src/objective-c/GRPCClient/GRPCCall+Interceptor.h +++ b/src/objective-c/GRPCClient/GRPCCall+Interceptor.h @@ -24,8 +24,15 @@ @interface GRPCCall2 (Interceptor) -+ (void)registerGlobalInterceptor:(id)interceptorFactory; +/** + * Register a global interceptor's factory in the current process. Only one interceptor can be + * registered in a process. If another one attempts to be registered, an exception will be raised. + */ ++ (void)registerGlobalInterceptor:(nonnull id)interceptorFactory; -+ (id)globalInterceptorFactory; +/** + * Get the global interceptor's factory. + */ ++ (nullable id)globalInterceptorFactory; @end diff --git a/src/objective-c/GRPCClient/GRPCCall+Interceptor.m b/src/objective-c/GRPCClient/GRPCCall+Interceptor.m index b1cad75d4f3..eed402f9c7e 100644 --- a/src/objective-c/GRPCClient/GRPCCall+Interceptor.m +++ b/src/objective-c/GRPCClient/GRPCCall+Interceptor.m @@ -26,18 +26,16 @@ static dispatch_once_t onceToken; @implementation GRPCCall2 (Interceptor) + (void)registerGlobalInterceptor:(id)interceptorFactory { + if (interceptorFactory == nil) { + return; + } dispatch_once(&onceToken, ^{ globalInterceptorLock = [[NSLock alloc] init]; }); [globalInterceptorLock lock]; - NSAssert(globalInterceptorFactory == nil, - @"Global interceptor is already registered. Only one global interceptor can be " - @"registered in a process"); if (globalInterceptorFactory != nil) { - NSLog( - @"Global interceptor is already registered. Only one global interceptor can be registered " - @"in a process"); [globalInterceptorLock unlock]; + [NSException raise:NSInternalInconsistencyException format:@"Global interceptor is already registered. Only one global interceptor can be registered in a process."]; return; } From a9ad58e0374ab83b511c6be5be45ae7f75f74b95 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 18 Jun 2019 10:13:13 -0700 Subject: [PATCH 08/55] Fix global interceptor tests --- src/objective-c/GRPCClient/GRPCCall.m | 48 +++---- .../tests/InteropTests/InteropTests.m | 133 ++++++++++++++---- 2 files changed, 125 insertions(+), 56 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 5280f8d01f4..c25e8daaf4c 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -155,11 +155,7 @@ const char *kCFStreamVarName = "grpc_cfstream"; [[GRPCInterceptorManager alloc] initWithNextInterceptor:nextInterceptor]; GRPCInterceptor *interceptor = [globalInterceptorFactory createInterceptorWithManager:manager]; - NSAssert(interceptor != nil, @"Failed to create interceptor from factory: %@", - globalInterceptorFactory); - if (interceptor == nil) { - NSLog(@"Failed to create interceptor from factory: %@", globalInterceptorFactory); - } else { + if (interceptor != nil) { [internalCall setResponseHandler:interceptor]; nextInterceptor = interceptor; nextManager = manager; @@ -168,35 +164,31 @@ const char *kCFStreamVarName = "grpc_cfstream"; // Finally initialize the interceptors in the chain NSArray *interceptorFactories = _actualCallOptions.interceptorFactories; - if (interceptorFactories.count == 0) { + for (int i = (int)interceptorFactories.count - 1; i >= 0; i--) { + GRPCInterceptorManager *manager = + [[GRPCInterceptorManager alloc] initWithNextInterceptor:nextInterceptor]; + GRPCInterceptor *interceptor = + [interceptorFactories[i] createInterceptorWithManager:manager]; + NSAssert(interceptor != nil, @"Failed to create interceptor from factory: %@", + interceptorFactories[i]); + if (interceptor == nil) { + NSLog(@"Failed to create interceptor from factory: %@", interceptorFactories[i]); + continue; + } if (nextManager == nil) { - [internalCall setResponseHandler:_responseHandler]; + [internalCall setResponseHandler:interceptor]; } else { - [nextManager setPreviousInterceptor:_responseHandler]; + [nextManager setPreviousInterceptor:interceptor]; } + nextInterceptor = interceptor; + nextManager = manager; + } + if (nextManager == nil) { + [internalCall setResponseHandler:_responseHandler]; } else { - for (int i = (int)interceptorFactories.count - 1; i >= 0; i--) { - GRPCInterceptorManager *manager = - [[GRPCInterceptorManager alloc] initWithNextInterceptor:nextInterceptor]; - GRPCInterceptor *interceptor = - [interceptorFactories[i] createInterceptorWithManager:manager]; - NSAssert(interceptor != nil, @"Failed to create interceptor from factory: %@", - interceptorFactories[i]); - if (interceptor == nil) { - NSLog(@"Failed to create interceptor from factory: %@", interceptorFactories[i]); - continue; - } - if (nextManager == nil) { - [internalCall setResponseHandler:interceptor]; - } else { - [nextManager setPreviousInterceptor:interceptor]; - } - nextInterceptor = interceptor; - nextManager = manager; - } - [nextManager setPreviousInterceptor:_responseHandler]; } + _firstInterceptor = nextInterceptor; } diff --git a/src/objective-c/tests/InteropTests/InteropTests.m b/src/objective-c/tests/InteropTests/InteropTests.m index 2c54fb80f3d..235fcaed483 100644 --- a/src/objective-c/tests/InteropTests/InteropTests.m +++ b/src/objective-c/tests/InteropTests/InteropTests.m @@ -121,7 +121,7 @@ initWithRequestDispatchQueue:(dispatch_queue_t)requestDispatchQueue @end -@interface HookIntercetpor : GRPCInterceptor +@interface HookInterceptor : GRPCInterceptor - (instancetype) initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager @@ -144,6 +144,7 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager @end @implementation HookInterceptorFactory { +@protected void (^_startHook)(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, GRPCInterceptorManager *manager); void (^_writeDataHook)(id data, GRPCInterceptorManager *manager); @@ -190,7 +191,7 @@ initWithRequestDispatchQueue:(dispatch_queue_t)requestDispatchQueue } - (GRPCInterceptor *)createInterceptorWithManager:(GRPCInterceptorManager *)interceptorManager { - return [[HookIntercetpor alloc] initWithInterceptorManager:interceptorManager + return [[HookInterceptor alloc] initWithInterceptorManager:interceptorManager requestDispatchQueue:_requestDispatchQueue responseDispatchQueue:_responseDispatchQueue startHook:_startHook @@ -205,7 +206,7 @@ initWithRequestDispatchQueue:(dispatch_queue_t)requestDispatchQueue @end -@implementation HookIntercetpor { +@implementation HookInterceptor { void (^_startHook)(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, GRPCInterceptorManager *manager); void (^_writeDataHook)(id data, GRPCInterceptorManager *manager); @@ -315,6 +316,92 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager @end +@interface GlobalInterceptorFactory : HookInterceptorFactory + +@property BOOL enabled; + +- (instancetype)initWithRequestDispatchQueue:(dispatch_queue_t)requestDispatchQueue + responseDispatchQueue:(dispatch_queue_t)responseDispatchQueue; + +- (void)setStartHook:(void (^)(GRPCRequestOptions *requestOptions, + GRPCCallOptions *callOptions, + GRPCInterceptorManager *manager))startHook + writeDataHook:(void (^)(id data, GRPCInterceptorManager *manager))writeDataHook + finishHook:(void (^)(GRPCInterceptorManager *manager))finishHook +receiveNextMessagesHook:(void (^)(NSUInteger numberOfMessages, + GRPCInterceptorManager *manager))receiveNextMessagesHook + responseHeaderHook:(void (^)(NSDictionary *initialMetadata, + GRPCInterceptorManager *manager))responseHeaderHook + responseDataHook:(void (^)(id data, GRPCInterceptorManager *manager))responseDataHook + responseCloseHook:(void (^)(NSDictionary *trailingMetadata, NSError *error, + GRPCInterceptorManager *manager))responseCloseHook + didWriteDataHook:(void (^)(GRPCInterceptorManager *manager))didWriteDataHook; + +@end + +@implementation GlobalInterceptorFactory + +- (instancetype)initWithRequestDispatchQueue:(dispatch_queue_t)requestDispatchQueue + responseDispatchQueue:(dispatch_queue_t)responseDispatchQueue { + _enabled = NO; + return [super initWithRequestDispatchQueue:requestDispatchQueue + responseDispatchQueue:responseDispatchQueue + startHook:nil + writeDataHook:nil + finishHook:nil + receiveNextMessagesHook:nil + responseHeaderHook:nil + responseDataHook:nil + responseCloseHook:nil + didWriteDataHook:nil]; +} + +- (GRPCInterceptor *)createInterceptorWithManager:(GRPCInterceptorManager *)interceptorManager { + if (_enabled) { + return [[HookInterceptor alloc] initWithInterceptorManager:interceptorManager + requestDispatchQueue:_requestDispatchQueue + responseDispatchQueue:_responseDispatchQueue + startHook:_startHook + writeDataHook:_writeDataHook + finishHook:_finishHook + receiveNextMessagesHook:_receiveNextMessagesHook + responseHeaderHook:_responseHeaderHook + responseDataHook:_responseDataHook + responseCloseHook:_responseCloseHook + didWriteDataHook:_didWriteDataHook]; + } else { + return nil; + } +} + +- (void)setStartHook:(void (^)(GRPCRequestOptions *requestOptions, + GRPCCallOptions *callOptions, + GRPCInterceptorManager *manager))startHook + writeDataHook:(void (^)(id data, GRPCInterceptorManager *manager))writeDataHook + finishHook:(void (^)(GRPCInterceptorManager *manager))finishHook +receiveNextMessagesHook:(void (^)(NSUInteger numberOfMessages, + GRPCInterceptorManager *manager))receiveNextMessagesHook + responseHeaderHook:(void (^)(NSDictionary *initialMetadata, + GRPCInterceptorManager *manager))responseHeaderHook + responseDataHook:(void (^)(id data, GRPCInterceptorManager *manager))responseDataHook + responseCloseHook:(void (^)(NSDictionary *trailingMetadata, NSError *error, + GRPCInterceptorManager *manager))responseCloseHook + didWriteDataHook:(void (^)(GRPCInterceptorManager *manager))didWriteDataHook { + _startHook = startHook; + _writeDataHook = writeDataHook; + _finishHook = finishHook; + _receiveNextMessagesHook = receiveNextMessagesHook; + _responseHeaderHook = responseHeaderHook; + _responseDataHook = responseDataHook; + _responseCloseHook = responseCloseHook; + _didWriteDataHook = didWriteDataHook; +} + +@end + +static GlobalInterceptorFactory *globalInterceptorFactory = nil; +static dispatch_once_t initGlobalInterceptorFactory; + #pragma mark Tests @implementation InteropTests { @@ -358,6 +445,13 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager #ifdef GRPC_CFSTREAM setenv(kCFStreamVarName, "1", 1); #endif + + dispatch_once(&initGlobalInterceptorFactory, ^{ + dispatch_queue_t globalInterceptorQueue = dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL); + globalInterceptorFactory = [[GlobalInterceptorFactory alloc] initWithRequestDispatchQueue:globalInterceptorQueue + responseDispatchQueue:globalInterceptorQueue]; + [GRPCCall2 registerGlobalInterceptor:globalInterceptorFactory]; + }); } - (void)setUp { @@ -1531,10 +1625,7 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager __block NSUInteger responseDataCount = 0; __block NSUInteger responseCloseCount = 0; __block NSUInteger didWriteDataCount = 0; - id factory = [[HookInterceptorFactory alloc] - initWithRequestDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) - responseDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) - startHook:^(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, + [globalInterceptorFactory setStartHook:^(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, GRPCInterceptorManager *manager) { startCount++; XCTAssertEqualObjects(requestOptions.host, [[self class] host]); @@ -1585,7 +1676,7 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager options.PEMRootCertificates = [[self class] PEMRootCertificates]; options.hostNameOverride = [[self class] hostNameOverride]; options.flowControlEnabled = YES; - [GRPCCall2 registerGlobalInterceptor:factory]; + globalInterceptorFactory.enabled = YES; __block BOOL canWriteData = NO; __block GRPCStreamingProtoCall *call = [_service @@ -1631,6 +1722,7 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager XCTAssertEqual(responseDataCount, 4); XCTAssertEqual(responseCloseCount, 1); XCTAssertEqual(didWriteDataCount, 4); + globalInterceptorFactory.enabled = NO; } - (void)testConflictingGlobalInterceptors { @@ -1645,24 +1737,11 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager responseDataHook:nil responseCloseHook:nil didWriteDataHook:nil]; - id factory2 = [[HookInterceptorFactory alloc] - initWithRequestDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) - responseDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) - startHook:nil - writeDataHook:nil - finishHook:nil - receiveNextMessagesHook:nil - responseHeaderHook:nil - responseDataHook:nil - responseCloseHook:nil - didWriteDataHook:nil]; - - [GRPCCall2 registerGlobalInterceptor:factory]; @try { - [GRPCCall2 registerGlobalInterceptor:factory2]; + [GRPCCall2 registerGlobalInterceptor:factory]; XCTFail(@"Did not receive an exception when registering global interceptor the second time"); } @catch (NSException *exception) { - NSLog(@"Received exception as expected: %@", exception); + // Do nothing; test passes } } @@ -1731,10 +1810,7 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager __block NSUInteger globalResponseCloseCount = 0; __block NSUInteger globalDidWriteDataCount = 0; - id globalFactory = [[HookInterceptorFactory alloc] - initWithRequestDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) - responseDispatchQueue:dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL) - startHook:^(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, + [globalInterceptorFactory setStartHook:^(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, GRPCInterceptorManager *manager) { globalStartCount++; XCTAssertEqualObjects(requestOptions.host, [[self class] host]); @@ -1786,7 +1862,7 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager options.hostNameOverride = [[self class] hostNameOverride]; options.flowControlEnabled = YES; options.interceptorFactories = @[ factory ]; - [GRPCCall2 registerGlobalInterceptor:globalFactory]; + globalInterceptorFactory.enabled = YES; __block BOOL canWriteData = NO; __block GRPCStreamingProtoCall *call = [_service @@ -1834,6 +1910,7 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager XCTAssertEqual(globalResponseDataCount, 4); XCTAssertEqual(globalResponseCloseCount, 1); XCTAssertEqual(globalDidWriteDataCount, 4); + globalInterceptorFactory.enabled = NO; } @end From 9640bcddc24f9ec32e3ec967ca83986cf35814bf Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 18 Jun 2019 14:17:15 -0700 Subject: [PATCH 09/55] fix testHijackingInterceptor --- src/objective-c/tests/InteropTests/InteropTests.m | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/objective-c/tests/InteropTests/InteropTests.m b/src/objective-c/tests/InteropTests/InteropTests.m index 235fcaed483..5587e78bfae 100644 --- a/src/objective-c/tests/InteropTests/InteropTests.m +++ b/src/objective-c/tests/InteropTests/InteropTests.m @@ -1497,8 +1497,10 @@ static dispatch_once_t initGlobalInterceptorFactory; - (void)testHijackingInterceptor { NSUInteger kCancelAfterWrites = 2; XCTAssertNotNil([[self class] host]); - __weak XCTestExpectation *expectation = - [self expectationWithDescription:@"testHijackingInterceptor"]; + __weak XCTestExpectation *expectUserCallComplete = + [self expectationWithDescription:@"User call completed."]; + __weak XCTestExpectation *expectCallInternalComplete = + [self expectationWithDescription:@"Internal gRPC call completed."]; NSArray *responses = @[ @1, @2, @3, @4 ]; __block int index = 0; @@ -1555,6 +1557,7 @@ static dispatch_once_t initGlobalInterceptorFactory; XCTAssertNil(trailingMetadata); XCTAssertNotNil(error); XCTAssertEqual(error.code, GRPC_STATUS_CANCELLED); + [expectCallInternalComplete fulfill]; } didWriteDataHook:nil]; @@ -1596,7 +1599,7 @@ static dispatch_once_t initGlobalInterceptorFactory; XCTAssertEqual(index, 4, @"Received %i responses instead of 4.", index); - [expectation fulfill]; + [expectUserCallComplete fulfill]; }] callOptions:options]; [call start]; From ad61ae12359a0c493170499220e89530e6185c10 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 18 Jun 2019 14:23:06 -0700 Subject: [PATCH 10/55] clang-format --- .../GRPCClient/GRPCCall+Interceptor.m | 5 +- src/objective-c/GRPCClient/GRPCCall.m | 5 +- .../tests/InteropTests/InteropTests.m | 89 +++++++++---------- 3 files changed, 50 insertions(+), 49 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall+Interceptor.m b/src/objective-c/GRPCClient/GRPCCall+Interceptor.m index eed402f9c7e..0d9f1f6b9dc 100644 --- a/src/objective-c/GRPCClient/GRPCCall+Interceptor.m +++ b/src/objective-c/GRPCClient/GRPCCall+Interceptor.m @@ -35,7 +35,10 @@ static dispatch_once_t onceToken; [globalInterceptorLock lock]; if (globalInterceptorFactory != nil) { [globalInterceptorLock unlock]; - [NSException raise:NSInternalInconsistencyException format:@"Global interceptor is already registered. Only one global interceptor can be registered in a process."]; + [NSException raise:NSInternalInconsistencyException + format: + @"Global interceptor is already registered. Only one global interceptor can be " + @"registered in a process."]; return; } diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index c25e8daaf4c..d5f4e9a84da 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -166,9 +166,8 @@ const char *kCFStreamVarName = "grpc_cfstream"; NSArray *interceptorFactories = _actualCallOptions.interceptorFactories; for (int i = (int)interceptorFactories.count - 1; i >= 0; i--) { GRPCInterceptorManager *manager = - [[GRPCInterceptorManager alloc] initWithNextInterceptor:nextInterceptor]; - GRPCInterceptor *interceptor = - [interceptorFactories[i] createInterceptorWithManager:manager]; + [[GRPCInterceptorManager alloc] initWithNextInterceptor:nextInterceptor]; + GRPCInterceptor *interceptor = [interceptorFactories[i] createInterceptorWithManager:manager]; NSAssert(interceptor != nil, @"Failed to create interceptor from factory: %@", interceptorFactories[i]); if (interceptor == nil) { diff --git a/src/objective-c/tests/InteropTests/InteropTests.m b/src/objective-c/tests/InteropTests/InteropTests.m index 5587e78bfae..d8dc3f6c32d 100644 --- a/src/objective-c/tests/InteropTests/InteropTests.m +++ b/src/objective-c/tests/InteropTests/InteropTests.m @@ -144,7 +144,7 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager @end @implementation HookInterceptorFactory { -@protected + @protected void (^_startHook)(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, GRPCInterceptorManager *manager); void (^_writeDataHook)(id data, GRPCInterceptorManager *manager); @@ -323,19 +323,18 @@ initWithInterceptorManager:(GRPCInterceptorManager *)interceptorManager - (instancetype)initWithRequestDispatchQueue:(dispatch_queue_t)requestDispatchQueue responseDispatchQueue:(dispatch_queue_t)responseDispatchQueue; -- (void)setStartHook:(void (^)(GRPCRequestOptions *requestOptions, - GRPCCallOptions *callOptions, +- (void)setStartHook:(void (^)(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, GRPCInterceptorManager *manager))startHook - writeDataHook:(void (^)(id data, GRPCInterceptorManager *manager))writeDataHook - finishHook:(void (^)(GRPCInterceptorManager *manager))finishHook -receiveNextMessagesHook:(void (^)(NSUInteger numberOfMessages, - GRPCInterceptorManager *manager))receiveNextMessagesHook - responseHeaderHook:(void (^)(NSDictionary *initialMetadata, - GRPCInterceptorManager *manager))responseHeaderHook - responseDataHook:(void (^)(id data, GRPCInterceptorManager *manager))responseDataHook - responseCloseHook:(void (^)(NSDictionary *trailingMetadata, NSError *error, - GRPCInterceptorManager *manager))responseCloseHook - didWriteDataHook:(void (^)(GRPCInterceptorManager *manager))didWriteDataHook; + writeDataHook:(void (^)(id data, GRPCInterceptorManager *manager))writeDataHook + finishHook:(void (^)(GRPCInterceptorManager *manager))finishHook + receiveNextMessagesHook:(void (^)(NSUInteger numberOfMessages, + GRPCInterceptorManager *manager))receiveNextMessagesHook + responseHeaderHook:(void (^)(NSDictionary *initialMetadata, + GRPCInterceptorManager *manager))responseHeaderHook + responseDataHook:(void (^)(id data, GRPCInterceptorManager *manager))responseDataHook + responseCloseHook:(void (^)(NSDictionary *trailingMetadata, NSError *error, + GRPCInterceptorManager *manager))responseCloseHook + didWriteDataHook:(void (^)(GRPCInterceptorManager *manager))didWriteDataHook; @end @@ -374,19 +373,18 @@ receiveNextMessagesHook:(void (^)(NSUInteger numberOfMessages, } } -- (void)setStartHook:(void (^)(GRPCRequestOptions *requestOptions, - GRPCCallOptions *callOptions, +- (void)setStartHook:(void (^)(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, GRPCInterceptorManager *manager))startHook - writeDataHook:(void (^)(id data, GRPCInterceptorManager *manager))writeDataHook - finishHook:(void (^)(GRPCInterceptorManager *manager))finishHook -receiveNextMessagesHook:(void (^)(NSUInteger numberOfMessages, - GRPCInterceptorManager *manager))receiveNextMessagesHook - responseHeaderHook:(void (^)(NSDictionary *initialMetadata, - GRPCInterceptorManager *manager))responseHeaderHook - responseDataHook:(void (^)(id data, GRPCInterceptorManager *manager))responseDataHook - responseCloseHook:(void (^)(NSDictionary *trailingMetadata, NSError *error, - GRPCInterceptorManager *manager))responseCloseHook - didWriteDataHook:(void (^)(GRPCInterceptorManager *manager))didWriteDataHook { + writeDataHook:(void (^)(id data, GRPCInterceptorManager *manager))writeDataHook + finishHook:(void (^)(GRPCInterceptorManager *manager))finishHook + receiveNextMessagesHook:(void (^)(NSUInteger numberOfMessages, + GRPCInterceptorManager *manager))receiveNextMessagesHook + responseHeaderHook:(void (^)(NSDictionary *initialMetadata, + GRPCInterceptorManager *manager))responseHeaderHook + responseDataHook:(void (^)(id data, GRPCInterceptorManager *manager))responseDataHook + responseCloseHook:(void (^)(NSDictionary *trailingMetadata, NSError *error, + GRPCInterceptorManager *manager))responseCloseHook + didWriteDataHook:(void (^)(GRPCInterceptorManager *manager))didWriteDataHook { _startHook = startHook; _writeDataHook = writeDataHook; _finishHook = finishHook; @@ -448,8 +446,9 @@ static dispatch_once_t initGlobalInterceptorFactory; dispatch_once(&initGlobalInterceptorFactory, ^{ dispatch_queue_t globalInterceptorQueue = dispatch_queue_create(NULL, DISPATCH_QUEUE_SERIAL); - globalInterceptorFactory = [[GlobalInterceptorFactory alloc] initWithRequestDispatchQueue:globalInterceptorQueue - responseDispatchQueue:globalInterceptorQueue]; + globalInterceptorFactory = + [[GlobalInterceptorFactory alloc] initWithRequestDispatchQueue:globalInterceptorQueue + responseDispatchQueue:globalInterceptorQueue]; [GRPCCall2 registerGlobalInterceptor:globalInterceptorFactory]; }); } @@ -1628,15 +1627,15 @@ static dispatch_once_t initGlobalInterceptorFactory; __block NSUInteger responseDataCount = 0; __block NSUInteger responseCloseCount = 0; __block NSUInteger didWriteDataCount = 0; - [globalInterceptorFactory setStartHook:^(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, - GRPCInterceptorManager *manager) { - startCount++; - XCTAssertEqualObjects(requestOptions.host, [[self class] host]); - XCTAssertEqualObjects(requestOptions.path, @"/grpc.testing.TestService/FullDuplexCall"); - XCTAssertEqual(requestOptions.safety, GRPCCallSafetyDefault); - [manager startNextInterceptorWithRequest:[requestOptions copy] - callOptions:[callOptions copy]]; - } + [globalInterceptorFactory setStartHook:^(GRPCRequestOptions *requestOptions, + GRPCCallOptions *callOptions, + GRPCInterceptorManager *manager) { + startCount++; + XCTAssertEqualObjects(requestOptions.host, [[self class] host]); + XCTAssertEqualObjects(requestOptions.path, @"/grpc.testing.TestService/FullDuplexCall"); + XCTAssertEqual(requestOptions.safety, GRPCCallSafetyDefault); + [manager startNextInterceptorWithRequest:[requestOptions copy] callOptions:[callOptions copy]]; + } writeDataHook:^(id data, GRPCInterceptorManager *manager) { writeDataCount++; [manager writeNextInterceptorWithData:data]; @@ -1813,15 +1812,15 @@ static dispatch_once_t initGlobalInterceptorFactory; __block NSUInteger globalResponseCloseCount = 0; __block NSUInteger globalDidWriteDataCount = 0; - [globalInterceptorFactory setStartHook:^(GRPCRequestOptions *requestOptions, GRPCCallOptions *callOptions, - GRPCInterceptorManager *manager) { - globalStartCount++; - XCTAssertEqualObjects(requestOptions.host, [[self class] host]); - XCTAssertEqualObjects(requestOptions.path, @"/grpc.testing.TestService/FullDuplexCall"); - XCTAssertEqual(requestOptions.safety, GRPCCallSafetyDefault); - [manager startNextInterceptorWithRequest:[requestOptions copy] - callOptions:[callOptions copy]]; - } + [globalInterceptorFactory setStartHook:^(GRPCRequestOptions *requestOptions, + GRPCCallOptions *callOptions, + GRPCInterceptorManager *manager) { + globalStartCount++; + XCTAssertEqualObjects(requestOptions.host, [[self class] host]); + XCTAssertEqualObjects(requestOptions.path, @"/grpc.testing.TestService/FullDuplexCall"); + XCTAssertEqual(requestOptions.safety, GRPCCallSafetyDefault); + [manager startNextInterceptorWithRequest:[requestOptions copy] callOptions:[callOptions copy]]; + } writeDataHook:^(id data, GRPCInterceptorManager *manager) { globalWriteDataCount++; [manager writeNextInterceptorWithData:data]; From 6f315691da46b12d1b1728fd8643e36e2de97a78 Mon Sep 17 00:00:00 2001 From: mgravell Date: Tue, 2 Jul 2019 13:57:50 +0100 Subject: [PATCH 11/55] remove boxing of Timespec caused by Equals usage --- src/csharp/Grpc.Core/Internal/Timespec.cs | 44 ++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Internal/Timespec.cs b/src/csharp/Grpc.Core/Internal/Timespec.cs index 71628d50642..ac7b2f720fe 100644 --- a/src/csharp/Grpc.Core/Internal/Timespec.cs +++ b/src/csharp/Grpc.Core/Internal/Timespec.cs @@ -25,8 +25,50 @@ namespace Grpc.Core.Internal /// gpr_timespec from grpc/support/time.h /// [StructLayout(LayoutKind.Sequential)] - internal struct Timespec + internal struct Timespec : IEquatable { + /// + /// Indicates whether this instance and a specified object are equal. + /// + public override bool Equals(object obj) + { + return obj is Timespec && Equals((Timespec)obj); + } + + /// + /// Returns the hash code for this instance. + /// + public override int GetHashCode() + { + unchecked + { + const int Prime = 373587911; + int i = (int)clock_type; + i = (i * Prime) ^ tv_nsec; + i = (i * Prime) ^ tv_nsec.GetHashCode(); + return i; + } + } + + /// + /// Returns the full type name of this instance. + /// + public override string ToString() + { + base.ToString(); + return typeof(Timespec).FullName; + } + + /// + /// Indicates whether this instance and a specified object are equal. + /// + public bool Equals(Timespec other) + { + return this.clock_type == other.clock_type + && this.tv_nsec == other.tv_nsec + && this.tv_sec == other.tv_sec; + } + const long NanosPerSecond = 1000 * 1000 * 1000; const long NanosPerTick = 100; const long TicksPerSecond = NanosPerSecond / NanosPerTick; From 746287111d207445289b6cd06a03ecf662dd3e16 Mon Sep 17 00:00:00 2001 From: mgravell Date: Tue, 2 Jul 2019 14:05:50 +0100 Subject: [PATCH 12/55] (left a base-call in that I'd used to get the intellisense comment) --- src/csharp/Grpc.Core/Internal/Timespec.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/csharp/Grpc.Core/Internal/Timespec.cs b/src/csharp/Grpc.Core/Internal/Timespec.cs index ac7b2f720fe..0edaffb9f49 100644 --- a/src/csharp/Grpc.Core/Internal/Timespec.cs +++ b/src/csharp/Grpc.Core/Internal/Timespec.cs @@ -55,7 +55,6 @@ namespace Grpc.Core.Internal /// public override string ToString() { - base.ToString(); return typeof(Timespec).FullName; } From f757028ea41b7d6ddb46de4da6e3030da0bdbceb Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 2 Jul 2019 11:54:57 -0700 Subject: [PATCH 13/55] Sched combiner closures instead of running to avoid data races --- .../transport/chttp2/transport/chttp2_transport.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 48c3d002bbd..b05a48a6a0c 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1232,10 +1232,10 @@ static grpc_closure* add_closure_barrier(grpc_closure* closure) { return closure; } -static void null_then_run_closure(grpc_closure** closure, grpc_error* error) { +static void null_then_sched_closure(grpc_closure** closure, grpc_error* error) { grpc_closure* c = *closure; *closure = nullptr; - GRPC_CLOSURE_RUN(c, error); + GRPC_CLOSURE_SCHED(c, error); } void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t, @@ -1902,7 +1902,7 @@ void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_chttp2_transport* t, } grpc_chttp2_incoming_metadata_buffer_publish(&s->metadata_buffer[0], s->recv_initial_metadata); - null_then_run_closure(&s->recv_initial_metadata_ready, GRPC_ERROR_NONE); + null_then_sched_closure(&s->recv_initial_metadata_ready, GRPC_ERROR_NONE); } } @@ -1982,10 +1982,10 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, s->unprocessed_incoming_frames_buffer_cached_length = s->unprocessed_incoming_frames_buffer.length; if (error == GRPC_ERROR_NONE && *s->recv_message != nullptr) { - null_then_run_closure(&s->recv_message_ready, GRPC_ERROR_NONE); + null_then_sched_closure(&s->recv_message_ready, GRPC_ERROR_NONE); } else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) { *s->recv_message = nullptr; - null_then_run_closure(&s->recv_message_ready, GRPC_ERROR_NONE); + null_then_sched_closure(&s->recv_message_ready, GRPC_ERROR_NONE); } GRPC_ERROR_UNREF(error); } @@ -2051,8 +2051,8 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_chttp2_transport* t, s->collecting_stats = nullptr; grpc_chttp2_incoming_metadata_buffer_publish(&s->metadata_buffer[1], s->recv_trailing_metadata); - null_then_run_closure(&s->recv_trailing_metadata_finished, - GRPC_ERROR_NONE); + null_then_sched_closure(&s->recv_trailing_metadata_finished, + GRPC_ERROR_NONE); } } } From 3bce424458ed4a3fefd191349dda1e672d8df5d0 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 2 Jul 2019 15:00:32 -0700 Subject: [PATCH 14/55] Remove debug code --- src/core/lib/gpr/sync_posix.cc | 129 ---------------------------- src/core/lib/iomgr/timer_manager.cc | 44 ---------- 2 files changed, 173 deletions(-) diff --git a/src/core/lib/gpr/sync_posix.cc b/src/core/lib/gpr/sync_posix.cc index a30e36c11ac..fe2d5a9ca38 100644 --- a/src/core/lib/gpr/sync_posix.cc +++ b/src/core/lib/gpr/sync_posix.cc @@ -29,44 +29,6 @@ #include #include "src/core/lib/profiling/timers.h" -// For debug of the timer manager crash only. -// TODO (mxyan): remove after bug is fixed. -#ifdef GRPC_DEBUG_TIMER_MANAGER -#include -void (*g_grpc_debug_timer_manager_stats)( - int64_t timer_manager_init_count, int64_t timer_manager_shutdown_count, - int64_t fork_count, int64_t timer_wait_err, int64_t timer_cv_value, - int64_t timer_mu_value, int64_t abstime_sec_value, - int64_t abstime_nsec_value, int64_t abs_deadline_sec_value, - int64_t abs_deadline_nsec_value, int64_t now1_sec_value, - int64_t now1_nsec_value, int64_t now2_sec_value, int64_t now2_nsec_value, - int64_t add_result_sec_value, int64_t add_result_nsec_value, - int64_t sub_result_sec_value, int64_t sub_result_nsec_value, - int64_t next_value, int64_t start_time_sec, - int64_t start_time_nsec) = nullptr; -int64_t g_timer_manager_init_count = 0; -int64_t g_timer_manager_shutdown_count = 0; -int64_t g_fork_count = 0; -int64_t g_timer_wait_err = 0; -int64_t g_timer_cv_value = 0; -int64_t g_timer_mu_value = 0; -int64_t g_abstime_sec_value = -1; -int64_t g_abstime_nsec_value = -1; -int64_t g_abs_deadline_sec_value = -1; -int64_t g_abs_deadline_nsec_value = -1; -int64_t g_now1_sec_value = -1; -int64_t g_now1_nsec_value = -1; -int64_t g_now2_sec_value = -1; -int64_t g_now2_nsec_value = -1; -int64_t g_add_result_sec_value = -1; -int64_t g_add_result_nsec_value = -1; -int64_t g_sub_result_sec_value = -1; -int64_t g_sub_result_nsec_value = -1; -int64_t g_next_value = -1; -int64_t g_start_time_sec = -1; -int64_t g_start_time_nsec = -1; -#endif // GRPC_DEBUG_TIMER_MANAGER - #ifdef GPR_LOW_LEVEL_COUNTERS gpr_atm gpr_mu_locks = 0; gpr_atm gpr_counter_atm_cas = 0; @@ -152,63 +114,12 @@ void gpr_cv_destroy(gpr_cv* cv) { #endif } -// For debug of the timer manager crash only. -// TODO (mxyan): remove after bug is fixed. -#ifdef GRPC_DEBUG_TIMER_MANAGER -static gpr_timespec gpr_convert_clock_type_debug_timespec( - gpr_timespec t, gpr_clock_type clock_type, gpr_timespec& now1, - gpr_timespec& now2, gpr_timespec& add_result, gpr_timespec& sub_result) { - if (t.clock_type == clock_type) { - return t; - } - - if (t.tv_sec == INT64_MAX || t.tv_sec == INT64_MIN) { - t.clock_type = clock_type; - return t; - } - - if (clock_type == GPR_TIMESPAN) { - return gpr_time_sub(t, gpr_now(t.clock_type)); - } - - if (t.clock_type == GPR_TIMESPAN) { - return gpr_time_add(gpr_now(clock_type), t); - } - - now1 = gpr_now(t.clock_type); - sub_result = gpr_time_sub(t, now1); - now2 = gpr_now(clock_type); - add_result = gpr_time_add(now2, sub_result); - return add_result; -} - -#define gpr_convert_clock_type_debug(t, clock_type, now1, now2, add_result, \ - sub_result) \ - gpr_convert_clock_type_debug_timespec((t), (clock_type), (now1), (now2), \ - (add_result), (sub_result)) -#else #define gpr_convert_clock_type_debug(t, clock_type, now1, now2, add_result, \ sub_result) \ gpr_convert_clock_type((t), (clock_type)) -#endif int gpr_cv_wait(gpr_cv* cv, gpr_mu* mu, gpr_timespec abs_deadline) { int err = 0; -#ifdef GRPC_DEBUG_TIMER_MANAGER - // For debug of the timer manager crash only. - // TODO (mxyan): remove after bug is fixed. - gpr_timespec abs_deadline_copy; - abs_deadline_copy.tv_sec = abs_deadline.tv_sec; - abs_deadline_copy.tv_nsec = abs_deadline.tv_nsec; - gpr_timespec now1; - gpr_timespec now2; - gpr_timespec add_result; - gpr_timespec sub_result; - memset(&now1, 0, sizeof(now1)); - memset(&now2, 0, sizeof(now2)); - memset(&add_result, 0, sizeof(add_result)); - memset(&sub_result, 0, sizeof(sub_result)); -#endif if (gpr_time_cmp(abs_deadline, gpr_inf_future(abs_deadline.clock_type)) == 0) { #ifdef GRPC_ASAN_ENABLED @@ -232,47 +143,7 @@ int gpr_cv_wait(gpr_cv* cv, gpr_mu* mu, gpr_timespec abs_deadline) { #else err = pthread_cond_timedwait(cv, mu, &abs_deadline_ts); #endif - -#ifdef GRPC_DEBUG_TIMER_MANAGER - // For debug of the timer manager crash only. - // TODO (mxyan): remove after bug is fixed. - if (GPR_UNLIKELY(!(err == 0 || err == ETIMEDOUT || err == EAGAIN))) { - g_abstime_sec_value = abs_deadline_ts.tv_sec; - g_abstime_nsec_value = abs_deadline_ts.tv_nsec; - } -#endif } - -#ifdef GRPC_DEBUG_TIMER_MANAGER - // For debug of the timer manager crash only. - // TODO (mxyan): remove after bug is fixed. - if (GPR_UNLIKELY(!(err == 0 || err == ETIMEDOUT || err == EAGAIN))) { - if (g_grpc_debug_timer_manager_stats) { - g_timer_wait_err = err; - g_timer_cv_value = (int64_t)cv; - g_timer_mu_value = (int64_t)mu; - g_abs_deadline_sec_value = abs_deadline_copy.tv_sec; - g_abs_deadline_nsec_value = abs_deadline_copy.tv_nsec; - g_now1_sec_value = now1.tv_sec; - g_now1_nsec_value = now1.tv_nsec; - g_now2_sec_value = now2.tv_sec; - g_now2_nsec_value = now2.tv_nsec; - g_add_result_sec_value = add_result.tv_sec; - g_add_result_nsec_value = add_result.tv_nsec; - g_sub_result_sec_value = sub_result.tv_sec; - g_sub_result_nsec_value = sub_result.tv_nsec; - g_grpc_debug_timer_manager_stats( - g_timer_manager_init_count, g_timer_manager_shutdown_count, - g_fork_count, g_timer_wait_err, g_timer_cv_value, g_timer_mu_value, - g_abstime_sec_value, g_abstime_nsec_value, g_abs_deadline_sec_value, - g_abs_deadline_nsec_value, g_now1_sec_value, g_now1_nsec_value, - g_now2_sec_value, g_now2_nsec_value, g_add_result_sec_value, - g_add_result_nsec_value, g_sub_result_sec_value, - g_sub_result_nsec_value, g_next_value, g_start_time_sec, - g_start_time_nsec); - } - } -#endif GPR_ASSERT(err == 0 || err == ETIMEDOUT || err == EAGAIN); return err == ETIMEDOUT; } diff --git a/src/core/lib/iomgr/timer_manager.cc b/src/core/lib/iomgr/timer_manager.cc index bdf54909b4c..bb270af8129 100644 --- a/src/core/lib/iomgr/timer_manager.cc +++ b/src/core/lib/iomgr/timer_manager.cc @@ -61,30 +61,6 @@ static uint64_t g_timed_waiter_generation; static void timer_thread(void* completed_thread_ptr); -// For debug of the timer manager crash only. -// TODO (mxyan): remove after bug is fixed. -#ifdef GRPC_DEBUG_TIMER_MANAGER -extern int64_t g_timer_manager_init_count; -extern int64_t g_timer_manager_shutdown_count; -extern int64_t g_fork_count; -extern int64_t g_next_value; -#endif // GRPC_DEBUG_TIMER_MANAGER - -static void gc_completed_threads(void) { - if (g_completed_threads != nullptr) { - completed_thread* to_gc = g_completed_threads; - g_completed_threads = nullptr; - gpr_mu_unlock(&g_mu); - while (to_gc != nullptr) { - to_gc->thd.Join(); - completed_thread* next = to_gc->next; - gpr_free(to_gc); - to_gc = next; - } - gpr_mu_lock(&g_mu); - } -} - static void start_timer_thread_and_unlock(void) { GPR_ASSERT(g_threaded); ++g_waiter_count; @@ -203,11 +179,6 @@ static bool wait_until(grpc_millis next) { gpr_log(GPR_INFO, "sleep until kicked"); } - // For debug of the timer manager crash only. - // TODO (mxyan): remove after bug is fixed. -#ifdef GRPC_DEBUG_TIMER_MANAGER - g_next_value = next; -#endif gpr_cv_wait(&g_cv_wait, &g_mu, grpc_millis_to_timespec(next, GPR_CLOCK_MONOTONIC)); @@ -309,11 +280,6 @@ static void start_threads(void) { void grpc_timer_manager_init(void) { gpr_mu_init(&g_mu); gpr_cv_init(&g_cv_wait); -#ifdef GRPC_DEBUG_TIMER_MANAGER - // For debug of the timer manager crash only. - // TODO (mxyan): remove after bug is fixed. - g_timer_manager_init_count++; -#endif gpr_cv_init(&g_cv_shutdown); g_threaded = false; g_thread_count = 0; @@ -349,11 +315,6 @@ static void stop_threads(void) { } void grpc_timer_manager_shutdown(void) { -#ifdef GRPC_DEBUG_TIMER_MANAGER - // For debug of the timer manager crash only. - // TODO (mxyan): remove after bug is fixed. - g_timer_manager_shutdown_count++; -#endif stop_threads(); gpr_mu_destroy(&g_mu); @@ -362,11 +323,6 @@ void grpc_timer_manager_shutdown(void) { } void grpc_timer_manager_set_threading(bool threaded) { -#ifdef GRPC_DEBUG_TIMER_MANAGER - // For debug of the timer manager crash only. - // TODO (mxyan): remove after bug is fixed. - g_fork_count++; -#endif if (threaded) { start_threads(); } else { From 0a1f644da59a1f8598580d925e847c425e7a3786 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 2 Jul 2019 16:44:47 -0700 Subject: [PATCH 15/55] Revert undesired deletion --- src/core/lib/iomgr/timer_manager.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/core/lib/iomgr/timer_manager.cc b/src/core/lib/iomgr/timer_manager.cc index bb270af8129..04d4f881ecf 100644 --- a/src/core/lib/iomgr/timer_manager.cc +++ b/src/core/lib/iomgr/timer_manager.cc @@ -61,6 +61,21 @@ static uint64_t g_timed_waiter_generation; static void timer_thread(void* completed_thread_ptr); +static void gc_completed_threads(void) { + if (g_completed_threads != nullptr) { + completed_thread* to_gc = g_completed_threads; + g_completed_threads = nullptr; + gpr_mu_unlock(&g_mu); + while (to_gc != nullptr) { + to_gc->thd.Join(); + completed_thread* next = to_gc->next; + gpr_free(to_gc); + to_gc = next; + } + gpr_mu_lock(&g_mu); + } +} + static void start_timer_thread_and_unlock(void) { GPR_ASSERT(g_threaded); ++g_waiter_count; From aaf5cf4cb7604030b3e5c8d81ec62d9dff2606ed Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 3 Jul 2019 08:52:51 -0700 Subject: [PATCH 16/55] clang-format --- src/core/lib/iomgr/timer_manager.cc | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/core/lib/iomgr/timer_manager.cc b/src/core/lib/iomgr/timer_manager.cc index 04d4f881ecf..17cae5cd4c3 100644 --- a/src/core/lib/iomgr/timer_manager.cc +++ b/src/core/lib/iomgr/timer_manager.cc @@ -61,20 +61,20 @@ static uint64_t g_timed_waiter_generation; static void timer_thread(void* completed_thread_ptr); -static void gc_completed_threads(void) { - if (g_completed_threads != nullptr) { - completed_thread* to_gc = g_completed_threads; - g_completed_threads = nullptr; - gpr_mu_unlock(&g_mu); - while (to_gc != nullptr) { - to_gc->thd.Join(); - completed_thread* next = to_gc->next; - gpr_free(to_gc); - to_gc = next; - } - gpr_mu_lock(&g_mu); - } -} +static void gc_completed_threads(void) { + if (g_completed_threads != nullptr) { + completed_thread* to_gc = g_completed_threads; + g_completed_threads = nullptr; + gpr_mu_unlock(&g_mu); + while (to_gc != nullptr) { + to_gc->thd.Join(); + completed_thread* next = to_gc->next; + gpr_free(to_gc); + to_gc = next; + } + gpr_mu_lock(&g_mu); + } +} static void start_timer_thread_and_unlock(void) { GPR_ASSERT(g_threaded); From b41ded289e12e7fc96222fd1dbd7a508c164aac1 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 3 Jul 2019 12:35:12 -0700 Subject: [PATCH 17/55] WIP. Check for NULL --- src/core/lib/iomgr/fork_posix.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/fork_posix.cc b/src/core/lib/iomgr/fork_posix.cc index 629b08162fb..444d2574c7e 100644 --- a/src/core/lib/iomgr/fork_posix.cc +++ b/src/core/lib/iomgr/fork_posix.cc @@ -59,8 +59,9 @@ void grpc_prefork() { "environment variable GRPC_ENABLE_FORK_SUPPORT=1"); return; } - if (strcmp(grpc_get_poll_strategy_name(), "epoll1") != 0 && - strcmp(grpc_get_poll_strategy_name(), "poll") != 0) { + const char * poll_strategy_name = grpc_get_poll_strategy_name(); + if (poll_strategy_name == nullptr || strcmp(poll_strategy_name, "epoll1") != 0 && + strcmp(poll_strategy_name, "poll") != 0) { gpr_log(GPR_INFO, "Fork support is only compatible with the epoll1 and poll polling " "strategies"); From eab6f7a64b31226f73535bca97ca14c1c14b421c Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 3 Jul 2019 16:02:29 -0700 Subject: [PATCH 18/55] Clang format --- src/core/lib/iomgr/fork_posix.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/fork_posix.cc b/src/core/lib/iomgr/fork_posix.cc index 444d2574c7e..248e9ae5f54 100644 --- a/src/core/lib/iomgr/fork_posix.cc +++ b/src/core/lib/iomgr/fork_posix.cc @@ -59,9 +59,10 @@ void grpc_prefork() { "environment variable GRPC_ENABLE_FORK_SUPPORT=1"); return; } - const char * poll_strategy_name = grpc_get_poll_strategy_name(); - if (poll_strategy_name == nullptr || strcmp(poll_strategy_name, "epoll1") != 0 && - strcmp(poll_strategy_name, "poll") != 0) { + const char* poll_strategy_name = grpc_get_poll_strategy_name(); + if (poll_strategy_name == nullptr || + strcmp(poll_strategy_name, "epoll1") != 0 && + strcmp(poll_strategy_name, "poll") != 0) { gpr_log(GPR_INFO, "Fork support is only compatible with the epoll1 and poll polling " "strategies"); From dd22893c322dcd7c4425d5a2aba26789fa4fbc23 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 3 Jul 2019 16:03:23 -0700 Subject: [PATCH 19/55] Clarify API contract for grpc_get_poll_strategy_name --- src/core/lib/iomgr/ev_posix.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc index d78ac2dac2d..02e4da1c349 100644 --- a/src/core/lib/iomgr/ev_posix.cc +++ b/src/core/lib/iomgr/ev_posix.cc @@ -206,7 +206,8 @@ void grpc_register_event_engine_factory(const char* name, GPR_ASSERT(false); } -/* Call this only after calling grpc_event_engine_init() */ +/*If grpc_event_engine_init() has been called, returns the poll_strategy_name. + * Otherwise, returns nullptr. */ const char* grpc_get_poll_strategy_name() { return g_poll_strategy_name; } void grpc_event_engine_init(void) { From 5087ab48bf5377c2dc89fe7b28b5dbbefab5889c Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 3 Jul 2019 16:22:40 -0700 Subject: [PATCH 20/55] Reenable signal handling test --- src/python/grpcio_tests/commands.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/python/grpcio_tests/commands.py b/src/python/grpcio_tests/commands.py index 166cea101a4..dc0795d4a12 100644 --- a/src/python/grpcio_tests/commands.py +++ b/src/python/grpcio_tests/commands.py @@ -145,8 +145,6 @@ class TestGevent(setuptools.Command): 'unit._exit_test.ExitTest.test_in_flight_partial_unary_stream_call', 'unit._exit_test.ExitTest.test_in_flight_partial_stream_unary_call', 'unit._exit_test.ExitTest.test_in_flight_partial_stream_stream_call', - # TODO(https://github.com/grpc/grpc/issues/18980): Reenable. - 'unit._signal_handling_test.SignalHandlingTest', 'unit._metadata_flags_test', 'health_check._health_servicer_test.HealthServicerTest.test_cancelled_watch_removed_from_watch_list', # TODO(https://github.com/grpc/grpc/issues/17330) enable these three tests From 6bcd74e903ac2e5491026e550dfcc744be27a411 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 3 Jul 2019 16:58:38 -0700 Subject: [PATCH 21/55] Add parentheses --- src/core/lib/iomgr/fork_posix.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/fork_posix.cc b/src/core/lib/iomgr/fork_posix.cc index 248e9ae5f54..e678b4f5905 100644 --- a/src/core/lib/iomgr/fork_posix.cc +++ b/src/core/lib/iomgr/fork_posix.cc @@ -61,8 +61,8 @@ void grpc_prefork() { } const char* poll_strategy_name = grpc_get_poll_strategy_name(); if (poll_strategy_name == nullptr || - strcmp(poll_strategy_name, "epoll1") != 0 && - strcmp(poll_strategy_name, "poll") != 0) { + (strcmp(poll_strategy_name, "epoll1") != 0 && + strcmp(poll_strategy_name, "poll") != 0)) { gpr_log(GPR_INFO, "Fork support is only compatible with the epoll1 and poll polling " "strategies"); From fbd5a47181020a1c83141b5295ac055e44949a56 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 5 Jul 2019 08:20:38 -0400 Subject: [PATCH 22/55] use System.Memory and Span<> on all TFMs --- .../Grpc.Core.Api/DeserializationContext.cs | 4 +--- src/csharp/Grpc.Core.Api/Grpc.Core.Api.csproj | 9 +------ .../Grpc.Core.Tests/Grpc.Core.Tests.csproj | 4 ---- .../DefaultDeserializationContextTest.cs | 14 ++--------- .../Internal/FakeBufferReaderManagerTest.cs | 2 +- .../Internal/ReusableSliceBufferTest.cs | 17 +++---------- .../Grpc.Core.Tests/Internal/SliceTest.cs | 21 ++-------------- src/csharp/Grpc.Core/Grpc.Core.csproj | 6 ++--- .../Internal/DefaultDeserializationContext.cs | 24 ++----------------- .../Grpc.Core/Internal/ReusableSliceBuffer.cs | 3 --- src/csharp/Grpc.Core/Internal/Slice.cs | 9 ------- 11 files changed, 14 insertions(+), 99 deletions(-) diff --git a/src/csharp/Grpc.Core.Api/DeserializationContext.cs b/src/csharp/Grpc.Core.Api/DeserializationContext.cs index 966bcfa8c8e..eb00f67601e 100644 --- a/src/csharp/Grpc.Core.Api/DeserializationContext.cs +++ b/src/csharp/Grpc.Core.Api/DeserializationContext.cs @@ -48,13 +48,12 @@ namespace Grpc.Core throw new NotImplementedException(); } -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY /// /// Gets the entire payload as a ReadOnlySequence. /// The ReadOnlySequence is only valid for the duration of the deserializer routine and the caller must not access it after the deserializer returns. /// Using the read only sequence is the most efficient way to access the message payload. Where possible it allows directly /// accessing the received payload without needing to perform any buffer copying or buffer allocations. - /// NOTE: This method is only available in the netstandard2.0 build of the library. + /// NOTE: In order to access the payload via this method, your compiler needs to support C# 7.2 (to be able to use the Span type). /// NOTE: Deserializers are expected not to call this method (or other payload accessor methods) more than once per received message /// (as there is no practical reason for doing so) and DeserializationContext implementations are free to assume so. /// @@ -63,6 +62,5 @@ namespace Grpc.Core { throw new NotImplementedException(); } -#endif } } diff --git a/src/csharp/Grpc.Core.Api/Grpc.Core.Api.csproj b/src/csharp/Grpc.Core.Api/Grpc.Core.Api.csproj index 0a958299ec8..a0d41a260da 100755 --- a/src/csharp/Grpc.Core.Api/Grpc.Core.Api.csproj +++ b/src/csharp/Grpc.Core.Api/Grpc.Core.Api.csproj @@ -20,18 +20,11 @@ true - - $(DefineConstants);GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY - - - - - - + diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 693646bea1c..1aa314f4aec 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -9,10 +9,6 @@ true - - $(DefineConstants);GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY - - diff --git a/src/csharp/Grpc.Core.Tests/Internal/DefaultDeserializationContextTest.cs b/src/csharp/Grpc.Core.Tests/Internal/DefaultDeserializationContextTest.cs index 63baab31624..8b635f9288f 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/DefaultDeserializationContextTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/DefaultDeserializationContextTest.cs @@ -17,18 +17,14 @@ #endregion using System; +using System.Buffers; using System.Collections.Generic; +using System.Runtime.InteropServices; using Grpc.Core; using Grpc.Core.Internal; using Grpc.Core.Utils; using NUnit.Framework; -using System.Runtime.InteropServices; - -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY -using System.Buffers; -#endif - namespace Grpc.Core.Internal.Tests { public class DefaultDeserializationContextTest @@ -47,7 +43,6 @@ namespace Grpc.Core.Internal.Tests fakeBufferReaderManager.Dispose(); } -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY [TestCase] public void PayloadAsReadOnlySequence_ZeroSegmentPayload() { @@ -118,7 +113,6 @@ namespace Grpc.Core.Internal.Tests Assert.IsFalse(segmentEnumerator.MoveNext()); } -#endif [TestCase] public void NullPayloadNotAllowed() @@ -196,9 +190,7 @@ namespace Grpc.Core.Internal.Tests // Getting payload multiple times is illegal Assert.Throws(typeof(InvalidOperationException), () => context.PayloadAsNewBuffer()); -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY Assert.Throws(typeof(InvalidOperationException), () => context.PayloadAsReadOnlySequence()); -#endif } [TestCase] @@ -215,9 +207,7 @@ namespace Grpc.Core.Internal.Tests Assert.AreEqual(0, context.PayloadLength); Assert.Throws(typeof(NullReferenceException), () => context.PayloadAsNewBuffer()); -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY Assert.Throws(typeof(NullReferenceException), () => context.PayloadAsReadOnlySequence()); -#endif // Previously reset context can be initialized again var origBuffer2 = GetTestBuffer(50); diff --git a/src/csharp/Grpc.Core.Tests/Internal/FakeBufferReaderManagerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/FakeBufferReaderManagerTest.cs index 7c4ff652bd3..8211b7fe00c 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/FakeBufferReaderManagerTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/FakeBufferReaderManagerTest.cs @@ -103,7 +103,7 @@ namespace Grpc.Core.Internal.Tests private void AssertSliceDataEqual(byte[] expected, Slice actual) { var actualSliceData = new byte[actual.Length]; - actual.CopyTo(new ArraySegment(actualSliceData)); + actual.ToSpanUnsafe().CopyTo(actualSliceData); CollectionAssert.AreEqual(expected, actualSliceData); } diff --git a/src/csharp/Grpc.Core.Tests/Internal/ReusableSliceBufferTest.cs b/src/csharp/Grpc.Core.Tests/Internal/ReusableSliceBufferTest.cs index 7630785aef4..bfe4f6451d0 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/ReusableSliceBufferTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/ReusableSliceBufferTest.cs @@ -17,19 +17,15 @@ #endregion using System; +using System.Buffers; using System.Collections.Generic; using System.Linq; +using System.Runtime.InteropServices; using Grpc.Core; using Grpc.Core.Internal; using Grpc.Core.Utils; using NUnit.Framework; -using System.Runtime.InteropServices; - -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY -using System.Buffers; -#endif - namespace Grpc.Core.Internal.Tests { // Converts IBufferReader into instances of ReadOnlySequence @@ -50,7 +46,6 @@ namespace Grpc.Core.Internal.Tests fakeBufferReaderManager.Dispose(); } -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY [TestCase] public void NullPayload() { @@ -131,13 +126,7 @@ namespace Grpc.Core.Internal.Tests } return result; } -#else - [TestCase] - public void OnlySupportedOnNetCore() - { - // Test case needs to exist to make C# sanity test happy. - } -#endif + private byte[] GetTestBuffer(int length) { var testBuffer = new byte[length]; diff --git a/src/csharp/Grpc.Core.Tests/Internal/SliceTest.cs b/src/csharp/Grpc.Core.Tests/Internal/SliceTest.cs index eb090bbfa50..ff4d74727b5 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/SliceTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/SliceTest.cs @@ -33,7 +33,7 @@ namespace Grpc.Core.Internal.Tests [TestCase(10)] [TestCase(100)] [TestCase(1000)] - public void SliceFromNativePtr_CopyToArraySegment(int bufferLength) + public void SliceFromNativePtr_Copy(int bufferLength) { var origBuffer = GetTestBuffer(bufferLength); var gcHandle = GCHandle.Alloc(origBuffer, GCHandleType.Pinned); @@ -43,7 +43,7 @@ namespace Grpc.Core.Internal.Tests Assert.AreEqual(bufferLength, slice.Length); var newBuffer = new byte[bufferLength]; - slice.CopyTo(new ArraySegment(newBuffer)); + slice.ToSpanUnsafe().CopyTo(newBuffer); CollectionAssert.AreEqual(origBuffer, newBuffer); } finally @@ -52,23 +52,6 @@ namespace Grpc.Core.Internal.Tests } } - [TestCase] - public void SliceFromNativePtr_CopyToArraySegmentTooSmall() - { - var origBuffer = GetTestBuffer(100); - var gcHandle = GCHandle.Alloc(origBuffer, GCHandleType.Pinned); - try - { - var slice = new Slice(gcHandle.AddrOfPinnedObject(), origBuffer.Length); - var tooSmall = new byte[origBuffer.Length - 1]; - Assert.Catch(typeof(ArgumentException), () => slice.CopyTo(new ArraySegment(tooSmall))); - } - finally - { - gcHandle.Free(); - } - } - // create a buffer of given size and fill it with some data private byte[] GetTestBuffer(int length) { diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 1844ce335bc..b7c7851a34b 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -23,9 +23,8 @@ true - + 7.2 - $(DefineConstants);GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY @@ -100,8 +99,7 @@ - - + diff --git a/src/csharp/Grpc.Core/Internal/DefaultDeserializationContext.cs b/src/csharp/Grpc.Core/Internal/DefaultDeserializationContext.cs index 946c37de190..b668ea18841 100644 --- a/src/csharp/Grpc.Core/Internal/DefaultDeserializationContext.cs +++ b/src/csharp/Grpc.Core/Internal/DefaultDeserializationContext.cs @@ -16,13 +16,10 @@ #endregion -using Grpc.Core.Utils; using System; -using System.Threading; - -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY using System.Buffers; -#endif +using System.Threading; +using Grpc.Core.Utils; namespace Grpc.Core.Internal { @@ -33,9 +30,7 @@ namespace Grpc.Core.Internal IBufferReader bufferReader; int payloadLength; -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY ReusableSliceBuffer cachedSliceBuffer = new ReusableSliceBuffer(); -#endif public DefaultDeserializationContext() { @@ -51,14 +46,12 @@ namespace Grpc.Core.Internal return buffer; } -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY public override ReadOnlySequence PayloadAsReadOnlySequence() { var sequence = cachedSliceBuffer.PopulateFrom(bufferReader); GrpcPreconditions.CheckState(sequence.Length == payloadLength); return sequence; } -#endif public void Initialize(IBufferReader bufferReader) { @@ -70,9 +63,7 @@ namespace Grpc.Core.Internal { this.bufferReader = null; this.payloadLength = 0; -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY this.cachedSliceBuffer.Invalidate(); -#endif } public static DefaultDeserializationContext GetInitializedThreadLocal(IBufferReader bufferReader) @@ -84,18 +75,7 @@ namespace Grpc.Core.Internal private void FillContinguousBuffer(IBufferReader reader, byte[] destination) { -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY PayloadAsReadOnlySequence().CopyTo(new Span(destination)); -#else - int offset = 0; - while (reader.TryGetNextSlice(out Slice slice)) - { - slice.CopyTo(new ArraySegment(destination, offset, (int)slice.Length)); - offset += (int)slice.Length; - } - // check that we filled the entire destination - GrpcPreconditions.CheckState(offset == payloadLength); -#endif } } } diff --git a/src/csharp/Grpc.Core/Internal/ReusableSliceBuffer.cs b/src/csharp/Grpc.Core/Internal/ReusableSliceBuffer.cs index 2d38509e511..078e59e9d1b 100644 --- a/src/csharp/Grpc.Core/Internal/ReusableSliceBuffer.cs +++ b/src/csharp/Grpc.Core/Internal/ReusableSliceBuffer.cs @@ -16,8 +16,6 @@ #endregion -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY - using Grpc.Core.Utils; using System; using System.Threading; @@ -145,4 +143,3 @@ namespace Grpc.Core.Internal } } } -#endif diff --git a/src/csharp/Grpc.Core/Internal/Slice.cs b/src/csharp/Grpc.Core/Internal/Slice.cs index 22eb9537951..b6e07014808 100644 --- a/src/csharp/Grpc.Core/Internal/Slice.cs +++ b/src/csharp/Grpc.Core/Internal/Slice.cs @@ -40,14 +40,6 @@ namespace Grpc.Core.Internal public int Length => length; - // copies data of the slice to given span. - // there needs to be enough space in the destination buffer - public void CopyTo(ArraySegment destination) - { - Marshal.Copy(dataPtr, destination.Array, destination.Offset, length); - } - -#if GRPC_CSHARP_SUPPORT_SYSTEM_MEMORY public Span ToSpanUnsafe() { unsafe @@ -55,7 +47,6 @@ namespace Grpc.Core.Internal return new Span((byte*) dataPtr, length); } } -#endif /// /// Returns a that represents the current . From 05772b699fe9496c41b516c221698e8610c45dd1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 5 Jul 2019 08:23:12 -0400 Subject: [PATCH 23/55] a bit of cleanup --- .../Grpc.Core/Internal/DefaultDeserializationContext.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/csharp/Grpc.Core/Internal/DefaultDeserializationContext.cs b/src/csharp/Grpc.Core/Internal/DefaultDeserializationContext.cs index b668ea18841..7b5997b3fed 100644 --- a/src/csharp/Grpc.Core/Internal/DefaultDeserializationContext.cs +++ b/src/csharp/Grpc.Core/Internal/DefaultDeserializationContext.cs @@ -42,7 +42,7 @@ namespace Grpc.Core.Internal public override byte[] PayloadAsNewBuffer() { var buffer = new byte[payloadLength]; - FillContinguousBuffer(bufferReader, buffer); + PayloadAsReadOnlySequence().CopyTo(buffer); return buffer; } @@ -72,10 +72,5 @@ namespace Grpc.Core.Internal instance.Initialize(bufferReader); return instance; } - - private void FillContinguousBuffer(IBufferReader reader, byte[] destination) - { - PayloadAsReadOnlySequence().CopyTo(new Span(destination)); - } } } From 1bada10afb3b82663c7c6007fcd0ed989a745a7a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 8 Jul 2019 12:00:37 -0400 Subject: [PATCH 24/55] add new dependencies to Unity package --- src/csharp/build_unitypackage.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 4211d70b235..06552e8464b 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -65,6 +65,9 @@ copy /Y nativelibs\csharp_ext_macos_ios\libgrpc.a unitypackage\unitypackage_skel @rem add gRPC dependencies @rem TODO(jtattermusch): also include XMLdoc copy /Y Grpc.Core\bin\Release\net45\System.Interactive.Async.dll unitypackage\unitypackage_skeleton\Plugins\System.Interactive.Async\lib\net45\System.Interactive.Async.dll || goto :error +copy /Y Grpc.Core\bin\Release\net45\System.Runtime.CompilerServices.Unsafe.dll unitypackage\unitypackage_skeleton\Plugins\System.Runtime.CompilerServices.Unsafe\lib\net45\System.Runtime.CompilerServices.Unsafe.dll || goto :error +copy /Y Grpc.Core\bin\Release\net45\System.Buffers.dll unitypackage\unitypackage_skeleton\Plugins\System.Buffers\lib\net45\System.Buffers.dll || goto :error +copy /Y Grpc.Core\bin\Release\net45\System.Memory.dll unitypackage\unitypackage_skeleton\Plugins\System.Memory\lib\net45\System.Memory.dll || goto :error @rem add Google.Protobuf @rem TODO(jtattermusch): also include XMLdoc From a933d3d00a7fd9d0a89e08977547778b8bb5ede0 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 8 Jul 2019 12:06:16 -0400 Subject: [PATCH 25/55] add unity package skeleton for newly added dependencies --- .../Plugins/System.Buffers/lib.meta | 10 ++++++ .../Plugins/System.Buffers/lib/net45.meta | 10 ++++++ .../lib/net45/System.Buffers.dll.meta | 32 +++++++++++++++++++ .../lib/net45/System.Buffers.xml.meta | 9 ++++++ .../Plugins/System.Memory/lib.meta | 10 ++++++ .../Plugins/System.Memory/lib/net45.meta | 10 ++++++ .../lib/net45/System.Memory.dll.meta | 32 +++++++++++++++++++ .../lib/net45/System.Memory.xml.meta | 9 ++++++ .../lib.meta | 10 ++++++ .../lib/net45.meta | 10 ++++++ ...m.Runtime.CompilerServices.Unsafe.dll.meta | 32 +++++++++++++++++++ ...m.Runtime.CompilerServices.Unsafe.xml.meta | 9 ++++++ 12 files changed, 183 insertions(+) create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.dll.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.xml.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.dll.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.xml.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.dll.meta create mode 100644 src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.xml.meta diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib.meta new file mode 100644 index 00000000000..d7ae012a397 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: bd5ddd2522dc301488ffe002106fe0ab +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45.meta new file mode 100644 index 00000000000..9b1748d3e76 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: a68443518bcd1d44ba88a871dcab0c83 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.dll.meta new file mode 100644 index 00000000000..bb910fe9229 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.dll.meta @@ -0,0 +1,32 @@ +fileFormatVersion: 2 +guid: 4c05e46cbf00c68408f5ddc1eef9ae3b +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.xml.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.xml.meta new file mode 100644 index 00000000000..53f91502bd6 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.xml.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 6fc38864f2d3dde46b3833c62c91a008 +timeCreated: 1531219386 +licenseType: Free +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib.meta new file mode 100644 index 00000000000..d7ae012a397 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: bd5ddd2522dc301488ffe002106fe0ab +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45.meta new file mode 100644 index 00000000000..9b1748d3e76 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: a68443518bcd1d44ba88a871dcab0c83 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.dll.meta new file mode 100644 index 00000000000..bb910fe9229 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.dll.meta @@ -0,0 +1,32 @@ +fileFormatVersion: 2 +guid: 4c05e46cbf00c68408f5ddc1eef9ae3b +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.xml.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.xml.meta new file mode 100644 index 00000000000..53f91502bd6 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.xml.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 6fc38864f2d3dde46b3833c62c91a008 +timeCreated: 1531219386 +licenseType: Free +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib.meta new file mode 100644 index 00000000000..d7ae012a397 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: bd5ddd2522dc301488ffe002106fe0ab +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45.meta new file mode 100644 index 00000000000..9b1748d3e76 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: a68443518bcd1d44ba88a871dcab0c83 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.dll.meta new file mode 100644 index 00000000000..bb910fe9229 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.dll.meta @@ -0,0 +1,32 @@ +fileFormatVersion: 2 +guid: 4c05e46cbf00c68408f5ddc1eef9ae3b +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.xml.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.xml.meta new file mode 100644 index 00000000000..53f91502bd6 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.xml.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 6fc38864f2d3dde46b3833c62c91a008 +timeCreated: 1531219386 +licenseType: Free +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: From 6a71664035fc0ac237bff396c58407901f0b7bd8 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 8 Jul 2019 14:41:44 -0400 Subject: [PATCH 26/55] fix sanity test --- templates/src/csharp/build_unitypackage.bat.template | 3 +++ 1 file changed, 3 insertions(+) diff --git a/templates/src/csharp/build_unitypackage.bat.template b/templates/src/csharp/build_unitypackage.bat.template index d6f2e3c7f0d..07f19c2af3e 100755 --- a/templates/src/csharp/build_unitypackage.bat.template +++ b/templates/src/csharp/build_unitypackage.bat.template @@ -67,6 +67,9 @@ @rem add gRPC dependencies @rem TODO(jtattermusch): also include XMLdoc copy /Y Grpc.Core\bin\Release\net45\System.Interactive.Async.dll unitypackage\unitypackage_skeleton\Plugins\System.Interactive.Async\lib\net45\System.Interactive.Async.dll || goto :error + copy /Y Grpc.Core\bin\Release\net45\System.Runtime.CompilerServices.Unsafe.dll unitypackage\unitypackage_skeleton\Plugins\System.Runtime.CompilerServices.Unsafe\lib\net45\System.Runtime.CompilerServices.Unsafe.dll || goto :error + copy /Y Grpc.Core\bin\Release\net45\System.Buffers.dll unitypackage\unitypackage_skeleton\Plugins\System.Buffers\lib\net45\System.Buffers.dll || goto :error + copy /Y Grpc.Core\bin\Release\net45\System.Memory.dll unitypackage\unitypackage_skeleton\Plugins\System.Memory\lib\net45\System.Memory.dll || goto :error @rem add Google.Protobuf @rem TODO(jtattermusch): also include XMLdoc From 11e60b8f30ca0bc77f6cec284806c5be2a33087b Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 9 Jul 2019 16:50:53 -0700 Subject: [PATCH 27/55] Add documentation for compression enums --- src/python/grpcio/grpc/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py index f0c198db66d..ac5192e0fed 100644 --- a/src/python/grpcio/grpc/__init__.py +++ b/src/python/grpcio/grpc/__init__.py @@ -1856,10 +1856,16 @@ def _create_servicer_context(rpc_event, state, request_deserializer): context._finalize_state() # pylint: disable=protected-access +@enum.unique class Compression(enum.IntEnum): """Indicates the compression method to be used for an RPC. This enumeration is part of an EXPERIMENTAL API. + + Attributes: + NoCompression: Do not use compression algorithm. + Deflate: Use "Deflate" compression algorithm. + Gzip: Use "Gzip" compression algorithm. """ NoCompression = _compression.NoCompression Deflate = _compression.Deflate From 51f80abb5a80f7b2666b88306740751ea90ca98e Mon Sep 17 00:00:00 2001 From: Arjun Roy Date: Thu, 13 Jun 2019 17:09:53 -0700 Subject: [PATCH 28/55] Coalesced some grpc_slice_buffer_tiny_add calls in hpack_enc --- .../chttp2/transport/hpack_encoder.cc | 28 +++++++++++-------- src/core/lib/slice/slice_buffer.cc | 3 +- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.cc b/src/core/ext/transport/chttp2/transport/hpack_encoder.cc index 666d28975a8..2b40b7b03ed 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.cc @@ -381,10 +381,11 @@ static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor* c, uint32_t len_val_len; GPR_ASSERT(len_val <= UINT32_MAX); len_val_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)len_val, 1); - GRPC_CHTTP2_WRITE_VARINT(key_index, 2, 0x40, - add_tiny_header_data(st, len_pfx), len_pfx); + GPR_DEBUG_ASSERT(len_pfx + len_val_len < GRPC_SLICE_INLINED_SIZE); + uint8_t* data = add_tiny_header_data(st, len_pfx + len_val_len); + GRPC_CHTTP2_WRITE_VARINT(key_index, 2, 0x40, data, len_pfx); GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, value.huffman_prefix, - add_tiny_header_data(st, len_val_len), len_val_len); + &data[len_pfx], len_val_len); add_wire_value(st, value); } @@ -398,10 +399,11 @@ static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor* c, uint32_t len_val_len; GPR_ASSERT(len_val <= UINT32_MAX); len_val_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)len_val, 1); - GRPC_CHTTP2_WRITE_VARINT(key_index, 4, 0x00, - add_tiny_header_data(st, len_pfx), len_pfx); + GPR_DEBUG_ASSERT(len_pfx + len_val_len < GRPC_SLICE_INLINED_SIZE); + uint8_t* data = add_tiny_header_data(st, len_pfx + len_val_len); + GRPC_CHTTP2_WRITE_VARINT(key_index, 4, 0x00, data, len_pfx); GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, value.huffman_prefix, - add_tiny_header_data(st, len_val_len), len_val_len); + &data[len_pfx], len_val_len); add_wire_value(st, value); } @@ -418,9 +420,10 @@ static void emit_lithdr_incidx_v(grpc_chttp2_hpack_compressor* c, uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1); GPR_ASSERT(len_key <= UINT32_MAX); GPR_ASSERT(wire_value_length(value) <= UINT32_MAX); - *add_tiny_header_data(st, 1) = 0x40; - GRPC_CHTTP2_WRITE_VARINT(len_key, 1, 0x00, - add_tiny_header_data(st, len_key_len), len_key_len); + GPR_DEBUG_ASSERT(1 + len_key_len < GRPC_SLICE_INLINED_SIZE); + uint8_t* data = add_tiny_header_data(st, 1 + len_key_len); + data[0] = 0x40; + GRPC_CHTTP2_WRITE_VARINT(len_key, 1, 0x00, &data[1], len_key_len); add_header_data(st, grpc_slice_ref_internal(GRPC_MDKEY(elem))); GRPC_CHTTP2_WRITE_VARINT(len_val, 1, value.huffman_prefix, add_tiny_header_data(st, len_val_len), len_val_len); @@ -440,9 +443,10 @@ static void emit_lithdr_noidx_v(grpc_chttp2_hpack_compressor* c, uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1); GPR_ASSERT(len_key <= UINT32_MAX); GPR_ASSERT(wire_value_length(value) <= UINT32_MAX); - *add_tiny_header_data(st, 1) = 0x00; - GRPC_CHTTP2_WRITE_VARINT(len_key, 1, 0x00, - add_tiny_header_data(st, len_key_len), len_key_len); + /* Preconditions passed; emit header. */ + uint8_t* data = add_tiny_header_data(st, 1 + len_key_len); + data[0] = 0x00; + GRPC_CHTTP2_WRITE_VARINT(len_key, 1, 0x00, &data[1], len_key_len); add_header_data(st, grpc_slice_ref_internal(GRPC_MDKEY(elem))); GRPC_CHTTP2_WRITE_VARINT(len_val, 1, value.huffman_prefix, add_tiny_header_data(st, len_val_len), len_val_len); diff --git a/src/core/lib/slice/slice_buffer.cc b/src/core/lib/slice/slice_buffer.cc index 2b69f576a3c..dfa4b84fd0f 100644 --- a/src/core/lib/slice/slice_buffer.cc +++ b/src/core/lib/slice/slice_buffer.cc @@ -103,7 +103,7 @@ uint8_t* grpc_slice_buffer_tiny_add(grpc_slice_buffer* sb, size_t n) { sb->length += n; - if (sb->count == 0) goto add_new; + if (sb->count == 0) goto add_first; back = &sb->slices[sb->count - 1]; if (back->refcount) goto add_new; if ((back->data.inlined.length + n) > sizeof(back->data.inlined.bytes)) @@ -115,6 +115,7 @@ uint8_t* grpc_slice_buffer_tiny_add(grpc_slice_buffer* sb, size_t n) { add_new: maybe_embiggen(sb); +add_first: back = &sb->slices[sb->count]; sb->count++; back->refcount = nullptr; From 109edca9710ade3b67af6cf5a59cd48caf0da795 Mon Sep 17 00:00:00 2001 From: Julien Boeuf Date: Sun, 7 Jul 2019 22:44:33 -0700 Subject: [PATCH 29/55] Adding C++ API and implementation for STS credentials: - marked as experimental. - also changed the name of a field in the options struct. --- CMakeLists.txt | 75 +++++---- Makefile | 80 +++++---- build.yaml | 21 +-- include/grpc/grpc_security.h | 26 +-- include/grpcpp/security/credentials.h | 18 ++ include/grpcpp/security/credentials_impl.h | 64 +++++++ .../credentials/oauth2/oauth2_credentials.cc | 5 +- src/cpp/client/secure_credentials.cc | 148 +++++++++++++++++ src/cpp/client/secure_credentials.h | 11 ++ test/core/security/BUILD | 1 + test/core/security/fetch_oauth2.cc | 72 ++++---- test/cpp/client/credentials_test.cc | 157 ++++++++++++++++++ .../generated/sources_and_headers.json | 33 ++-- 13 files changed, 560 insertions(+), 151 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22c926df537..8dfa4e8009a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -332,7 +332,6 @@ add_dependencies(buildtests_c grpc_channel_stack_test) add_dependencies(buildtests_c grpc_completion_queue_test) add_dependencies(buildtests_c grpc_completion_queue_threading_test) add_dependencies(buildtests_c grpc_credentials_test) -add_dependencies(buildtests_c grpc_fetch_oauth2) add_dependencies(buildtests_c grpc_ipv6_loopback_available_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c grpc_json_token_test) @@ -634,6 +633,7 @@ add_dependencies(buildtests_cxx golden_file_test) add_dependencies(buildtests_cxx grpc_alts_credentials_options_test) add_dependencies(buildtests_cxx grpc_cli) add_dependencies(buildtests_cxx grpc_core_map_test) +add_dependencies(buildtests_cxx grpc_fetch_oauth2) add_dependencies(buildtests_cxx grpc_linux_system_roots_test) add_dependencies(buildtests_cxx grpc_tool_test) add_dependencies(buildtests_cxx grpclb_api_test) @@ -8270,40 +8270,6 @@ target_link_libraries(grpc_credentials_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -add_executable(grpc_fetch_oauth2 - test/core/security/fetch_oauth2.cc -) - - -target_include_directories(grpc_fetch_oauth2 - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include - PRIVATE ${_gRPC_SSL_INCLUDE_DIR} - PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} - PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} - PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} - PRIVATE ${_gRPC_CARES_INCLUDE_DIR} - PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} - PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} - PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} -) - -target_link_libraries(grpc_fetch_oauth2 - ${_gRPC_ALLTARGETS_LIBRARIES} - grpc_test_util - grpc - gpr -) - - # avoid dependency on libstdc++ - if (_gRPC_CORE_NOSTDCXX_FLAGS) - set_target_properties(grpc_fetch_oauth2 PROPERTIES LINKER_LANGUAGE C) - target_compile_options(grpc_fetch_oauth2 PRIVATE $<$:${_gRPC_CORE_NOSTDCXX_FLAGS}>) - endif() - -endif (gRPC_BUILD_TESTS) -if (gRPC_BUILD_TESTS) - add_executable(grpc_ipv6_loopback_available_test test/core/iomgr/grpc_ipv6_loopback_available_test.cc ) @@ -14080,6 +14046,45 @@ endif() endif (gRPC_BUILD_CODEGEN) if (gRPC_BUILD_TESTS) +add_executable(grpc_fetch_oauth2 + test/core/security/fetch_oauth2.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(grpc_fetch_oauth2 + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(grpc_fetch_oauth2 + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc++ + grpc + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(grpc_linux_system_roots_test test/core/security/linux_system_roots_test.cc third_party/googletest/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index c11462f419a..2a1bfad1f35 100644 --- a/Makefile +++ b/Makefile @@ -1056,7 +1056,6 @@ grpc_completion_queue_test: $(BINDIR)/$(CONFIG)/grpc_completion_queue_test grpc_completion_queue_threading_test: $(BINDIR)/$(CONFIG)/grpc_completion_queue_threading_test grpc_create_jwt: $(BINDIR)/$(CONFIG)/grpc_create_jwt grpc_credentials_test: $(BINDIR)/$(CONFIG)/grpc_credentials_test -grpc_fetch_oauth2: $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 grpc_ipv6_loopback_available_test: $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test grpc_json_token_test: $(BINDIR)/$(CONFIG)/grpc_json_token_test grpc_jwt_verifier_test: $(BINDIR)/$(CONFIG)/grpc_jwt_verifier_test @@ -1219,6 +1218,7 @@ grpc_cli: $(BINDIR)/$(CONFIG)/grpc_cli grpc_core_map_test: $(BINDIR)/$(CONFIG)/grpc_core_map_test grpc_cpp_plugin: $(BINDIR)/$(CONFIG)/grpc_cpp_plugin grpc_csharp_plugin: $(BINDIR)/$(CONFIG)/grpc_csharp_plugin +grpc_fetch_oauth2: $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 grpc_linux_system_roots_test: $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test grpc_node_plugin: $(BINDIR)/$(CONFIG)/grpc_node_plugin grpc_objective_c_plugin: $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin @@ -1489,7 +1489,6 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/grpc_completion_queue_test \ $(BINDIR)/$(CONFIG)/grpc_completion_queue_threading_test \ $(BINDIR)/$(CONFIG)/grpc_credentials_test \ - $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 \ $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test \ $(BINDIR)/$(CONFIG)/grpc_json_token_test \ $(BINDIR)/$(CONFIG)/grpc_jwt_verifier_test \ @@ -1693,6 +1692,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ $(BINDIR)/$(CONFIG)/grpc_core_map_test \ + $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 \ $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ @@ -1857,6 +1857,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ $(BINDIR)/$(CONFIG)/grpc_core_map_test \ + $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 \ $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ @@ -10987,38 +10988,6 @@ endif endif -GRPC_FETCH_OAUTH2_SRC = \ - test/core/security/fetch_oauth2.cc \ - -GRPC_FETCH_OAUTH2_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_FETCH_OAUTH2_SRC)))) -ifeq ($(NO_SECURE),true) - -# You can't build secure targets if you don't have OpenSSL. - -$(BINDIR)/$(CONFIG)/grpc_fetch_oauth2: openssl_dep_error - -else - - - -$(BINDIR)/$(CONFIG)/grpc_fetch_oauth2: $(GRPC_FETCH_OAUTH2_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a - $(E) "[LD] Linking $@" - $(Q) mkdir -p `dirname $@` - $(Q) $(LD) $(LDFLAGS) $(GRPC_FETCH_OAUTH2_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 - -endif - -$(OBJDIR)/$(CONFIG)/test/core/security/fetch_oauth2.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a - -deps_grpc_fetch_oauth2: $(GRPC_FETCH_OAUTH2_OBJS:.o=.dep) - -ifneq ($(NO_SECURE),true) -ifneq ($(NO_DEPS),true) --include $(GRPC_FETCH_OAUTH2_OBJS:.o=.dep) -endif -endif - - GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_SRC = \ test/core/iomgr/grpc_ipv6_loopback_available_test.cc \ @@ -17134,6 +17103,49 @@ ifneq ($(NO_DEPS),true) endif +GRPC_FETCH_OAUTH2_SRC = \ + test/core/security/fetch_oauth2.cc \ + +GRPC_FETCH_OAUTH2_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_FETCH_OAUTH2_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpc_fetch_oauth2: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+. + +$(BINDIR)/$(CONFIG)/grpc_fetch_oauth2: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/grpc_fetch_oauth2: $(PROTOBUF_DEP) $(GRPC_FETCH_OAUTH2_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_FETCH_OAUTH2_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/core/security/fetch_oauth2.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_grpc_fetch_oauth2: $(GRPC_FETCH_OAUTH2_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPC_FETCH_OAUTH2_OBJS:.o=.dep) +endif +endif + + GRPC_LINUX_SYSTEM_ROOTS_TEST_SRC = \ test/core/security/linux_system_roots_test.cc \ diff --git a/build.yaml b/build.yaml index 1c7be4e23ec..4134ecd8880 100644 --- a/build.yaml +++ b/build.yaml @@ -2863,16 +2863,6 @@ targets: - grpc_test_util - grpc - gpr -- name: grpc_fetch_oauth2 - build: test - run: false - language: c - src: - - test/core/security/fetch_oauth2.cc - deps: - - grpc_test_util - - grpc - - gpr - name: grpc_ipv6_loopback_available_test build: test language: c @@ -4945,6 +4935,17 @@ targets: deps: - grpc_plugin_support secure: false +- name: grpc_fetch_oauth2 + build: test + run: false + language: c++ + src: + - test/core/security/fetch_oauth2.cc + deps: + - grpc_test_util + - grpc++ + - grpc + - gpr - name: grpc_linux_system_roots_test gtest: true build: test diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index 8e4f26a2854..777142f38c6 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -330,20 +330,20 @@ GRPCAPI grpc_call_credentials* grpc_google_iam_credentials_create( /** Options for creating STS Oauth Token Exchange credentials following the IETF draft https://tools.ietf.org/html/draft-ietf-oauth-token-exchange-16. - Optional fields may be set to NULL. It is the responsibility of the caller to - ensure that the subject and actor tokens are refreshed on disk at the - specified paths. This API is used for experimental purposes for now and may - change in the future. */ + Optional fields may be set to NULL or empty string. It is the responsibility + of the caller to ensure that the subject and actor tokens are refreshed on + disk at the specified paths. This API is used for experimental purposes for + now and may change in the future. */ typedef struct { - const char* sts_endpoint_url; /* Required. */ - const char* resource; /* Optional. */ - const char* audience; /* Optional. */ - const char* scope; /* Optional. */ - const char* requested_token_type; /* Optional. */ - const char* subject_token_path; /* Required. */ - const char* subject_token_type; /* Required. */ - const char* actor_token_path; /* Optional. */ - const char* actor_token_type; /* Optional. */ + const char* token_exchange_service_uri; /* Required. */ + const char* resource; /* Optional. */ + const char* audience; /* Optional. */ + const char* scope; /* Optional. */ + const char* requested_token_type; /* Optional. */ + const char* subject_token_path; /* Required. */ + const char* subject_token_type; /* Required. */ + const char* actor_token_path; /* Optional. */ + const char* actor_token_type; /* Optional. */ } grpc_sts_credentials_options; /** Creates an STS credentials following the STS Token Exchanged specifed in the diff --git a/include/grpcpp/security/credentials.h b/include/grpcpp/security/credentials.h index b124d3d37be..5190b1b3393 100644 --- a/include/grpcpp/security/credentials.h +++ b/include/grpcpp/security/credentials.h @@ -106,6 +106,24 @@ MetadataCredentialsFromPlugin( namespace experimental { +typedef ::grpc_impl::experimental::StsCredentialsOptions StsCredentialsOptions; + +static inline grpc::Status StsCredentialsOptionsFromJson( + const grpc::string& json_string, StsCredentialsOptions* options) { + return ::grpc_impl::experimental::StsCredentialsOptionsFromJson(json_string, + options); +} + +static inline grpc::Status StsCredentialsOptionsFromEnv( + StsCredentialsOptions* options) { + return grpc_impl::experimental::StsCredentialsOptionsFromEnv(options); +} + +static inline std::shared_ptr StsCredentials( + const StsCredentialsOptions& options) { + return grpc_impl::experimental::StsCredentials(options); +} + typedef ::grpc_impl::experimental::AltsCredentialsOptions AltsCredentialsOptions; diff --git a/include/grpcpp/security/credentials_impl.h b/include/grpcpp/security/credentials_impl.h index 34920a55bbe..e236512f43e 100644 --- a/include/grpcpp/security/credentials_impl.h +++ b/include/grpcpp/security/credentials_impl.h @@ -259,6 +259,70 @@ std::shared_ptr MetadataCredentialsFromPlugin( namespace experimental { +/// Options for creating STS Oauth Token Exchange credentials following the IETF +/// draft https://tools.ietf.org/html/draft-ietf-oauth-token-exchange-16. +/// Optional fields may be set to empty string. It is the responsibility of the +/// caller to ensure that the subject and actor tokens are refreshed on disk at +/// the specified paths. +struct StsCredentialsOptions { + grpc::string token_exchange_service_uri; // Required. + grpc::string resource; // Optional. + grpc::string audience; // Optional. + grpc::string scope; // Optional. + grpc::string requested_token_type; // Optional. + grpc::string subject_token_path; // Required. + grpc::string subject_token_type; // Required. + grpc::string actor_token_path; // Optional. + grpc::string actor_token_type; // Optional. +}; + +/// Creates STS Options from a JSON string. The JSON schema is as follows: +/// { +/// "title": "STS Credentials Config", +/// "type": "object", +/// "required": ["token_exchange_service_uri", "subject_token_path", +/// "subject_token_type"], +/// "properties": { +/// "token_exchange_service_uri": { +/// "type": "string" +/// }, +/// "resource": { +/// "type": "string" +/// }, +/// "audience": { +/// "type": "string" +/// }, +/// "scope": { +/// "type": "string" +/// }, +/// "requested_token_type": { +/// "type": "string" +/// }, +/// "subject_token_path": { +/// "type": "string" +/// }, +/// "subject_token_type": { +/// "type": "string" +/// }, +/// "actor_token_path" : { +/// "type": "string" +/// }, +/// "actor_token_type": { +/// "type": "string" +/// } +/// } +/// } +grpc::Status StsCredentialsOptionsFromJson(const grpc::string& json_string, + StsCredentialsOptions* options); + +/// Creates STS credentials options from the $STS_CREDENTIALS environment +/// variable. This environment variable points to the path of a JSON file +/// comforming to the schema described above. +grpc::Status StsCredentialsOptionsFromEnv(StsCredentialsOptions* options); + +std::shared_ptr StsCredentials( + const StsCredentialsOptions& options); + /// Options used to build AltsCredentials. struct AltsCredentialsOptions { /// service accounts of target endpoint that will be acceptable diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc b/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc index 06fa8bbdb4b..7a584835b96 100644 --- a/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc +++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc @@ -18,6 +18,7 @@ #include +#include "src/core/lib/json/json.h" #include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h" #include @@ -641,8 +642,8 @@ grpc_error* ValidateStsCredentialsOptions( *sts_url_out = nullptr; InlinedVector error_list; UniquePtr sts_url( - options->sts_endpoint_url != nullptr - ? grpc_uri_parse(options->sts_endpoint_url, false) + options->token_exchange_service_uri != nullptr + ? grpc_uri_parse(options->token_exchange_service_uri, false) : nullptr); if (sts_url == nullptr) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc index d73b3e035c8..5de5a76194e 100644 --- a/src/cpp/client/secure_credentials.cc +++ b/src/cpp/client/secure_credentials.cc @@ -17,13 +17,23 @@ */ #include "src/cpp/client/secure_credentials.h" + +#include +#include #include #include #include +#include #include #include + +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/executor.h" +#include "src/core/lib/iomgr/load_file.h" +#include "src/core/lib/json/json.h" #include "src/core/lib/security/transport/auth_filters.h" +#include "src/core/lib/security/util/json_util.h" #include "src/cpp/client/create_channel_internal.h" #include "src/cpp/common/secure_auth_context.h" @@ -105,6 +115,144 @@ std::shared_ptr SslCredentials( namespace experimental { +namespace { + +void ClearStsCredentialsOptions(StsCredentialsOptions* options) { + if (options == nullptr) return; + options->token_exchange_service_uri.clear(); + options->resource.clear(); + options->audience.clear(); + options->scope.clear(); + options->requested_token_type.clear(); + options->subject_token_path.clear(); + options->subject_token_type.clear(); + options->actor_token_path.clear(); + options->actor_token_type.clear(); +} + +} // namespace + +// Builds STS credentials options from JSON. +grpc::Status StsCredentialsOptionsFromJson(const grpc::string& json_string, + StsCredentialsOptions* options) { + struct GrpcJsonDeleter { + void operator()(grpc_json* json) { grpc_json_destroy(json); } + }; + if (options == nullptr) { + return grpc::Status(grpc::INVALID_ARGUMENT, "options cannot be nullptr."); + } + ClearStsCredentialsOptions(options); + std::vector scratchpad(json_string.c_str(), + json_string.c_str() + json_string.size() + 1); + std::unique_ptr json( + grpc_json_parse_string(&scratchpad[0])); + if (json == nullptr) { + return grpc::Status(grpc::INVALID_ARGUMENT, "Invalid json."); + } + + // Required fields. + const char* value = grpc_json_get_string_property( + json.get(), "token_exchange_service_uri", nullptr); + if (value == nullptr) { + ClearStsCredentialsOptions(options); + return grpc::Status(grpc::INVALID_ARGUMENT, + "token_exchange_service_uri must be specified."); + } + options->token_exchange_service_uri.assign(value); + value = + grpc_json_get_string_property(json.get(), "subject_token_path", nullptr); + if (value == nullptr) { + ClearStsCredentialsOptions(options); + return grpc::Status(grpc::INVALID_ARGUMENT, + "subject_token_path must be specified."); + } + options->subject_token_path.assign(value); + value = + grpc_json_get_string_property(json.get(), "subject_token_type", nullptr); + if (value == nullptr) { + ClearStsCredentialsOptions(options); + return grpc::Status(grpc::INVALID_ARGUMENT, + "subject_token_type must be specified."); + } + options->subject_token_type.assign(value); + + // Optional fields. + value = grpc_json_get_string_property(json.get(), "resource", nullptr); + if (value != nullptr) options->resource.assign(value); + value = grpc_json_get_string_property(json.get(), "audience", nullptr); + if (value != nullptr) options->audience.assign(value); + value = grpc_json_get_string_property(json.get(), "scope", nullptr); + if (value != nullptr) options->scope.assign(value); + value = grpc_json_get_string_property(json.get(), "requested_token_type", + nullptr); + if (value != nullptr) options->requested_token_type.assign(value); + value = + grpc_json_get_string_property(json.get(), "actor_token_path", nullptr); + if (value != nullptr) options->actor_token_path.assign(value); + value = + grpc_json_get_string_property(json.get(), "actor_token_type", nullptr); + if (value != nullptr) options->actor_token_type.assign(value); + + return grpc::Status(); +} + +// Builds STS credentials Options from the $STS_CREDENTIALS env var. +grpc::Status StsCredentialsOptionsFromEnv(StsCredentialsOptions* options) { + if (options == nullptr) { + return grpc::Status(grpc::INVALID_ARGUMENT, "options cannot be nullptr."); + } + ClearStsCredentialsOptions(options); + grpc_slice json_string = grpc_empty_slice(); + char* sts_creds_path = gpr_getenv("STS_CREDENTIALS"); + grpc_error* error = GRPC_ERROR_NONE; + grpc::Status status; + auto cleanup = [&json_string, &sts_creds_path, &error, &status]() { + grpc_slice_unref_internal(json_string); + gpr_free(sts_creds_path); + GRPC_ERROR_UNREF(error); + return status; + }; + + if (sts_creds_path == nullptr) { + status = grpc::Status(grpc::NOT_FOUND, + "STS_CREDENTIALS environment variable not set."); + return cleanup(); + } + error = grpc_load_file(sts_creds_path, 1, &json_string); + if (error != GRPC_ERROR_NONE) { + status = grpc::Status(grpc::NOT_FOUND, grpc_error_string(error)); + return cleanup(); + } + status = StsCredentialsOptionsFromJson( + reinterpret_cast(GRPC_SLICE_START_PTR(json_string)), + options); + return cleanup(); +} + +// C++ to Core STS Credentials options. +grpc_sts_credentials_options StsCredentialsCppToCoreOptions( + const StsCredentialsOptions& options) { + grpc_sts_credentials_options opts; + memset(&opts, 0, sizeof(opts)); + opts.token_exchange_service_uri = options.token_exchange_service_uri.c_str(); + opts.resource = options.resource.c_str(); + opts.audience = options.audience.c_str(); + opts.scope = options.scope.c_str(); + opts.requested_token_type = options.requested_token_type.c_str(); + opts.subject_token_path = options.subject_token_path.c_str(); + opts.subject_token_type = options.subject_token_type.c_str(); + opts.actor_token_path = options.actor_token_path.c_str(); + opts.actor_token_type = options.actor_token_type.c_str(); + return opts; +} + +// Builds STS credentials. +std::shared_ptr StsCredentials( + const StsCredentialsOptions& options) { + auto opts = StsCredentialsCppToCoreOptions(options); + return WrapCallCredentials(grpc_sts_credentials_create(&opts, nullptr)); +} + // Builds ALTS Credentials given ALTS specific options std::shared_ptr AltsCredentials( const AltsCredentialsOptions& options) { diff --git a/src/cpp/client/secure_credentials.h b/src/cpp/client/secure_credentials.h index dd379ca657d..ed14df4938e 100644 --- a/src/cpp/client/secure_credentials.h +++ b/src/cpp/client/secure_credentials.h @@ -22,6 +22,7 @@ #include #include +#include #include #include "src/core/lib/security/credentials/credentials.h" @@ -68,6 +69,16 @@ class SecureCallCredentials final : public CallCredentials { grpc_call_credentials* const c_creds_; }; +namespace experimental { + +// Transforms C++ STS Credentials options to core options. The pointers of the +// resulting core options point to the memory held by the C++ options so C++ +// options need to be kept alive until after the core credentials creation. +grpc_sts_credentials_options StsCredentialsCppToCoreOptions( + const StsCredentialsOptions& options); + +} // namespace experimental + } // namespace grpc_impl namespace grpc { diff --git a/test/core/security/BUILD b/test/core/security/BUILD index d8dcdc25231..835c0ad7b65 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -171,6 +171,7 @@ grpc_cc_binary( ":oauth2_utils", "//:gpr", "//:grpc", + "//:grpc++", "//test/core/util:grpc_test_util", ], ) diff --git a/test/core/security/fetch_oauth2.cc b/test/core/security/fetch_oauth2.cc index d404368b8b9..1aa999758b0 100644 --- a/test/core/security/fetch_oauth2.cc +++ b/test/core/security/fetch_oauth2.cc @@ -26,53 +26,40 @@ #include #include +#include "grpcpp/security/credentials_impl.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/util/json_util.h" +#include "src/cpp/client/secure_credentials.h" #include "test/core/security/oauth2_utils.h" #include "test/core/util/cmdline.h" -static grpc_sts_credentials_options sts_options_from_json(grpc_json* json) { - grpc_sts_credentials_options options; - memset(&options, 0, sizeof(options)); - grpc_error* error = GRPC_ERROR_NONE; - options.sts_endpoint_url = - grpc_json_get_string_property(json, "sts_endpoint_url", &error); - GRPC_LOG_IF_ERROR("STS credentials parsing", error); - options.resource = grpc_json_get_string_property(json, "resource", nullptr); - options.audience = grpc_json_get_string_property(json, "audience", nullptr); - options.scope = grpc_json_get_string_property(json, "scope", nullptr); - options.requested_token_type = - grpc_json_get_string_property(json, "requested_token_type", nullptr); - options.subject_token_path = - grpc_json_get_string_property(json, "subject_token_path", &error); - GRPC_LOG_IF_ERROR("STS credentials parsing", error); - options.subject_token_type = - grpc_json_get_string_property(json, "subject_token_type", &error); - GRPC_LOG_IF_ERROR("STS credentials parsing", error); - options.actor_token_path = - grpc_json_get_string_property(json, "actor_token_path", nullptr); - options.actor_token_type = - grpc_json_get_string_property(json, "actor_token_type", nullptr); - return options; -} - static grpc_call_credentials* create_sts_creds(const char* json_file_path) { - grpc_slice sts_options_slice; - GPR_ASSERT(GRPC_LOG_IF_ERROR( - "load_file", grpc_load_file(json_file_path, 1, &sts_options_slice))); - grpc_json* json = grpc_json_parse_string( - reinterpret_cast(GRPC_SLICE_START_PTR(sts_options_slice))); - if (json == nullptr) { - gpr_log(GPR_ERROR, "Invalid json"); - return nullptr; + grpc_impl::experimental::StsCredentialsOptions options; + if (strlen(json_file_path) == 0) { + auto status = + grpc_impl::experimental::StsCredentialsOptionsFromEnv(&options); + if (!status.ok()) { + gpr_log(GPR_ERROR, "%s", status.error_message().c_str()); + return nullptr; + } + } else { + grpc_slice sts_options_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(json_file_path, 1, &sts_options_slice))); + auto status = grpc_impl::experimental::StsCredentialsOptionsFromJson( + reinterpret_cast(GRPC_SLICE_START_PTR(sts_options_slice)), + &options); + gpr_slice_unref(sts_options_slice); + if (!status.ok()) { + gpr_log(GPR_ERROR, "%s", status.error_message().c_str()); + return nullptr; + } } - grpc_sts_credentials_options options = sts_options_from_json(json); - grpc_call_credentials* result = - grpc_sts_credentials_create(&options, nullptr); - grpc_json_destroy(json); - gpr_slice_unref(sts_options_slice); + grpc_sts_credentials_options opts = + grpc_impl::experimental::StsCredentialsCppToCoreOptions(options); + grpc_call_credentials* result = grpc_sts_credentials_create(&opts, nullptr); return result; } @@ -99,9 +86,12 @@ int main(int argc, char** argv) { gpr_cmdline_add_string(cl, "json_refresh_token", "File path of the json refresh token.", &json_refresh_token_file_path); - gpr_cmdline_add_string(cl, "json_sts_options", - "File path of the json sts options.", - &json_sts_options_file_path); + gpr_cmdline_add_string( + cl, "json_sts_options", + "File path of the json sts options. If the path is empty, the program " + "will attempt to use the $STS_CREDENTIALS environment variable to access " + "a file containing the options.", + &json_sts_options_file_path); gpr_cmdline_add_flag( cl, "gce", "Get a token from the GCE metadata server (only works in GCE).", diff --git a/test/cpp/client/credentials_test.cc b/test/cpp/client/credentials_test.cc index e64e260a46c..d560fb6d8e8 100644 --- a/test/cpp/client/credentials_test.cc +++ b/test/cpp/client/credentials_test.cc @@ -20,9 +20,14 @@ #include +#include #include #include +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/gpr/tmpfile.h" +#include "src/cpp/client/secure_credentials.h" + namespace grpc { namespace testing { @@ -39,6 +44,158 @@ TEST_F(CredentialsTest, DefaultCredentials) { auto creds = GoogleDefaultCredentials(); } +TEST_F(CredentialsTest, StsCredentialsOptionsCppToCore) { + grpc::experimental::StsCredentialsOptions options; + options.token_exchange_service_uri = "https://foo.com/exchange"; + options.resource = "resource"; + options.audience = "audience"; + options.scope = "scope"; + // options.requested_token_type explicitly not set. + options.subject_token_path = "/foo/bar"; + options.subject_token_type = "nice_token_type"; + options.actor_token_path = "/foo/baz"; + options.actor_token_type = "even_nicer_token_type"; + grpc_sts_credentials_options core_opts = + grpc_impl::experimental::StsCredentialsCppToCoreOptions(options); + EXPECT_EQ(options.token_exchange_service_uri, + core_opts.token_exchange_service_uri); + EXPECT_EQ(options.resource, core_opts.resource); + EXPECT_EQ(options.audience, core_opts.audience); + EXPECT_EQ(options.scope, core_opts.scope); + EXPECT_EQ(options.requested_token_type, core_opts.requested_token_type); + EXPECT_EQ(options.subject_token_path, core_opts.subject_token_path); + EXPECT_EQ(options.subject_token_type, core_opts.subject_token_type); + EXPECT_EQ(options.actor_token_path, core_opts.actor_token_path); + EXPECT_EQ(options.actor_token_type, core_opts.actor_token_type); +} + +TEST_F(CredentialsTest, StsCredentialsOptionsJson) { + const char valid_json[] = R"( + { + "token_exchange_service_uri": "https://foo/exchange", + "resource": "resource", + "audience": "audience", + "scope": "scope", + "requested_token_type": "requested_token_type", + "subject_token_path": "subject_token_path", + "subject_token_type": "subject_token_type", + "actor_token_path": "actor_token_path", + "actor_token_type": "actor_token_type" + })"; + grpc::experimental::StsCredentialsOptions options; + EXPECT_TRUE( + grpc::experimental::StsCredentialsOptionsFromJson(valid_json, &options) + .ok()); + EXPECT_EQ(options.token_exchange_service_uri, "https://foo/exchange"); + EXPECT_EQ(options.resource, "resource"); + EXPECT_EQ(options.audience, "audience"); + EXPECT_EQ(options.scope, "scope"); + EXPECT_EQ(options.requested_token_type, "requested_token_type"); + EXPECT_EQ(options.subject_token_path, "subject_token_path"); + EXPECT_EQ(options.subject_token_type, "subject_token_type"); + EXPECT_EQ(options.actor_token_path, "actor_token_path"); + EXPECT_EQ(options.actor_token_type, "actor_token_type"); + + const char minimum_valid_json[] = R"( + { + "token_exchange_service_uri": "https://foo/exchange", + "subject_token_path": "subject_token_path", + "subject_token_type": "subject_token_type" + })"; + EXPECT_TRUE(grpc::experimental::StsCredentialsOptionsFromJson( + minimum_valid_json, &options) + .ok()); + EXPECT_EQ(options.token_exchange_service_uri, "https://foo/exchange"); + EXPECT_EQ(options.resource, ""); + EXPECT_EQ(options.audience, ""); + EXPECT_EQ(options.scope, ""); + EXPECT_EQ(options.requested_token_type, ""); + EXPECT_EQ(options.subject_token_path, "subject_token_path"); + EXPECT_EQ(options.subject_token_type, "subject_token_type"); + EXPECT_EQ(options.actor_token_path, ""); + EXPECT_EQ(options.actor_token_type, ""); + + const char invalid_json[] = R"( + I'm not a valid JSON. + )"; + EXPECT_EQ( + grpc::INVALID_ARGUMENT, + grpc::experimental::StsCredentialsOptionsFromJson(invalid_json, &options) + .error_code()); + + const char invalid_json_missing_subject_token_type[] = R"( + { + "token_exchange_service_uri": "https://foo/exchange", + "subject_token_path": "subject_token_path" + })"; + auto status = grpc::experimental::StsCredentialsOptionsFromJson( + invalid_json_missing_subject_token_type, &options); + EXPECT_EQ(grpc::INVALID_ARGUMENT, status.error_code()); + EXPECT_THAT(status.error_message(), + ::testing::HasSubstr("subject_token_type")); + + const char invalid_json_missing_subject_token_path[] = R"( + { + "token_exchange_service_uri": "https://foo/exchange", + "subject_token_type": "subject_token_type" + })"; + status = grpc::experimental::StsCredentialsOptionsFromJson( + invalid_json_missing_subject_token_path, &options); + EXPECT_EQ(grpc::INVALID_ARGUMENT, status.error_code()); + EXPECT_THAT(status.error_message(), + ::testing::HasSubstr("subject_token_path")); + + const char invalid_json_missing_token_exchange_uri[] = R"( + { + "subject_token_path": "subject_token_path", + "subject_token_type": "subject_token_type" + })"; + status = grpc::experimental::StsCredentialsOptionsFromJson( + invalid_json_missing_token_exchange_uri, &options); + EXPECT_EQ(grpc::INVALID_ARGUMENT, status.error_code()); + EXPECT_THAT(status.error_message(), + ::testing::HasSubstr("token_exchange_service_uri")); +} + +TEST_F(CredentialsTest, StsCredentialsOptionsFromEnv) { + // Unset env and check expected failure. + gpr_unsetenv("STS_CREDENTIALS"); + grpc::experimental::StsCredentialsOptions options; + auto status = grpc::experimental::StsCredentialsOptionsFromEnv(&options); + EXPECT_EQ(grpc::NOT_FOUND, status.error_code()); + + // Set env and check for success. + const char valid_json[] = R"( + { + "token_exchange_service_uri": "https://foo/exchange", + "subject_token_path": "subject_token_path", + "subject_token_type": "subject_token_type" + })"; + char* creds_file_name; + FILE* creds_file = gpr_tmpfile("sts_creds_options", &creds_file_name); + ASSERT_NE(creds_file_name, nullptr); + ASSERT_NE(creds_file, nullptr); + ASSERT_EQ(sizeof(valid_json), + fwrite(valid_json, 1, sizeof(valid_json), creds_file)); + fclose(creds_file); + gpr_setenv("STS_CREDENTIALS", creds_file_name); + gpr_free(creds_file_name); + status = grpc::experimental::StsCredentialsOptionsFromEnv(&options); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(options.token_exchange_service_uri, "https://foo/exchange"); + EXPECT_EQ(options.resource, ""); + EXPECT_EQ(options.audience, ""); + EXPECT_EQ(options.scope, ""); + EXPECT_EQ(options.requested_token_type, ""); + EXPECT_EQ(options.subject_token_path, "subject_token_path"); + EXPECT_EQ(options.subject_token_type, "subject_token_type"); + EXPECT_EQ(options.actor_token_path, ""); + EXPECT_EQ(options.actor_token_type, ""); + + // Cleanup. + gpr_unsetenv("STS_CREDENTIALS"); +} + } // namespace testing } // namespace grpc diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 92b3757c65e..0f560fd1063 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -1024,22 +1024,6 @@ "third_party": false, "type": "target" }, - { - "deps": [ - "gpr", - "grpc", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c", - "name": "grpc_fetch_oauth2", - "src": [ - "test/core/security/fetch_oauth2.cc" - ], - "third_party": false, - "type": "target" - }, { "deps": [ "gpr", @@ -3878,6 +3862,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "grpc", + "grpc++", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "grpc_fetch_oauth2", + "src": [ + "test/core/security/fetch_oauth2.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", From ee7b5be39649cee0b5b02a2f81a97794388a2c14 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 10 Jul 2019 04:35:30 -0400 Subject: [PATCH 30/55] more accurate comment --- src/csharp/Grpc.Core.Api/DeserializationContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csharp/Grpc.Core.Api/DeserializationContext.cs b/src/csharp/Grpc.Core.Api/DeserializationContext.cs index eb00f67601e..b0c3badbc24 100644 --- a/src/csharp/Grpc.Core.Api/DeserializationContext.cs +++ b/src/csharp/Grpc.Core.Api/DeserializationContext.cs @@ -53,7 +53,7 @@ namespace Grpc.Core /// The ReadOnlySequence is only valid for the duration of the deserializer routine and the caller must not access it after the deserializer returns. /// Using the read only sequence is the most efficient way to access the message payload. Where possible it allows directly /// accessing the received payload without needing to perform any buffer copying or buffer allocations. - /// NOTE: In order to access the payload via this method, your compiler needs to support C# 7.2 (to be able to use the Span type). + /// NOTE: When using this method, it is recommended to use C# 7.2 compiler to make it more useful (using Span type directly from your code requires C# 7.2)." /// NOTE: Deserializers are expected not to call this method (or other payload accessor methods) more than once per received message /// (as there is no practical reason for doing so) and DeserializationContext implementations are free to assume so. /// From 578f027e3c6363d2e0b5d9679a90e36a4bc50e55 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 10 Jul 2019 04:43:05 -0400 Subject: [PATCH 31/55] deduplicate GUIDs in unity package skeleton --- .../unitypackage_skeleton/Plugins/System.Buffers/lib.meta | 2 +- .../unitypackage_skeleton/Plugins/System.Buffers/lib/net45.meta | 2 +- .../Plugins/System.Buffers/lib/net45/System.Buffers.dll.meta | 2 +- .../Plugins/System.Buffers/lib/net45/System.Buffers.xml.meta | 2 +- .../unitypackage_skeleton/Plugins/System.Memory/lib.meta | 2 +- .../unitypackage_skeleton/Plugins/System.Memory/lib/net45.meta | 2 +- .../Plugins/System.Memory/lib/net45/System.Memory.dll.meta | 2 +- .../Plugins/System.Memory/lib/net45/System.Memory.xml.meta | 2 +- .../Plugins/System.Runtime.CompilerServices.Unsafe/lib.meta | 2 +- .../System.Runtime.CompilerServices.Unsafe/lib/net45.meta | 2 +- .../lib/net45/System.Runtime.CompilerServices.Unsafe.dll.meta | 2 +- .../lib/net45/System.Runtime.CompilerServices.Unsafe.xml.meta | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib.meta index d7ae012a397..754fcaee5f1 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: bd5ddd2522dc301488ffe002106fe0ab +guid: 0cb4be3dca2a49e6a920da037ea13d80 folderAsset: yes timeCreated: 1531219385 licenseType: Free diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45.meta index 9b1748d3e76..00368db2d4f 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: a68443518bcd1d44ba88a871dcab0c83 +guid: 53b3f7a608814da5a3e3207d10c85b4e folderAsset: yes timeCreated: 1531219385 licenseType: Free diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.dll.meta index bb910fe9229..6a9eae1c5b8 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.dll.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.dll.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 4c05e46cbf00c68408f5ddc1eef9ae3b +guid: bb037a236f584460af82c59c5d5ad972 timeCreated: 1531219386 licenseType: Free PluginImporter: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.xml.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.xml.meta index 53f91502bd6..14b3b367a4a 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.xml.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Buffers/lib/net45/System.Buffers.xml.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 6fc38864f2d3dde46b3833c62c91a008 +guid: 4b9fff86d3b2471eb0003735b3ce3a51 timeCreated: 1531219386 licenseType: Free TextScriptImporter: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib.meta index d7ae012a397..eab9851920f 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: bd5ddd2522dc301488ffe002106fe0ab +guid: 3d6c20bf92b74c03b1ba691cbce610e4 folderAsset: yes timeCreated: 1531219385 licenseType: Free diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45.meta index 9b1748d3e76..2d33fd95298 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: a68443518bcd1d44ba88a871dcab0c83 +guid: 7164a0a387b24d1a9d76f6d558fc44d2 folderAsset: yes timeCreated: 1531219385 licenseType: Free diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.dll.meta index bb910fe9229..11af5eae04c 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.dll.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.dll.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 4c05e46cbf00c68408f5ddc1eef9ae3b +guid: e774d51b8ba64a988dd37135e487105c timeCreated: 1531219386 licenseType: Free PluginImporter: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.xml.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.xml.meta index 53f91502bd6..4c04cedb0eb 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.xml.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Memory/lib/net45/System.Memory.xml.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 6fc38864f2d3dde46b3833c62c91a008 +guid: 3d27c5afe3114f67b0f6e27e36b89d5c timeCreated: 1531219386 licenseType: Free TextScriptImporter: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib.meta index d7ae012a397..b3a382aed11 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: bd5ddd2522dc301488ffe002106fe0ab +guid: f51cc0f065424fb2928eee7c2804bfd8 folderAsset: yes timeCreated: 1531219385 licenseType: Free diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45.meta index 9b1748d3e76..4154241974e 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: a68443518bcd1d44ba88a871dcab0c83 +guid: aad21c5c1f2f4c1391b1e4a475f163bb folderAsset: yes timeCreated: 1531219385 licenseType: Free diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.dll.meta index bb910fe9229..c3988cadd4e 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.dll.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.dll.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 4c05e46cbf00c68408f5ddc1eef9ae3b +guid: d378b9cd266a4a448f071c114e5f18cb timeCreated: 1531219386 licenseType: Free PluginImporter: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.xml.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.xml.meta index 53f91502bd6..e756ced26d0 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.xml.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Runtime.CompilerServices.Unsafe/lib/net45/System.Runtime.CompilerServices.Unsafe.xml.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 6fc38864f2d3dde46b3833c62c91a008 +guid: cfa8e546fee54a5ea3b20cf9dcf90fdf timeCreated: 1531219386 licenseType: Free TextScriptImporter: From 5dee89b06e3171ec1fee76245055106372d59e5b Mon Sep 17 00:00:00 2001 From: Wensheng Tang Date: Sun, 7 Jul 2019 01:59:28 +0800 Subject: [PATCH 32/55] Use grpc_error defs instead of NULL NULL is not clear for a grpc_error. Not sure we should use other error codes here. --- src/core/lib/iomgr/tcp_server_windows.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/iomgr/tcp_server_windows.cc b/src/core/lib/iomgr/tcp_server_windows.cc index abfa3be5708..0024f807ed3 100644 --- a/src/core/lib/iomgr/tcp_server_windows.cc +++ b/src/core/lib/iomgr/tcp_server_windows.cc @@ -409,7 +409,7 @@ static grpc_error* add_socket_to_server(grpc_tcp_server* s, SOCKET sock, gpr_log(GPR_ERROR, "on_connect error: %s", utf8_message); gpr_free(utf8_message); closesocket(sock); - return NULL; + return GRPC_ERROR_NONE; } error = prepare_socket(sock, addr, &port); From c01477360f148b67d7858b10ecb655aee2a24a31 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 3 Jul 2019 16:24:30 -0700 Subject: [PATCH 33/55] Add v1.22.0 to interop_matrix --- tools/interop_matrix/client_matrix.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index d6d0612b4e4..3f136b69bc2 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -98,6 +98,7 @@ LANG_RELEASE_MATRIX = { ('v1.19.0', ReleaseInfo(testcases_file='cxx__v1.0.1')), ('v1.20.0', ReleaseInfo()), ('v1.21.4', ReleaseInfo()), + ('v1.22.0', ReleaseInfo()), ]), 'go': OrderedDict( @@ -211,6 +212,7 @@ LANG_RELEASE_MATRIX = { ('v1.19.0', ReleaseInfo()), ('v1.20.0', ReleaseInfo()), ('v1.21.4', ReleaseInfo()), + ('v1.22.0', ReleaseInfo()), ]), 'node': OrderedDict([ @@ -260,6 +262,7 @@ LANG_RELEASE_MATRIX = { ('v1.19.0', ReleaseInfo()), ('v1.20.0', ReleaseInfo()), ('v1.21.4', ReleaseInfo()), + ('v1.22.0', ReleaseInfo()), # TODO: https://github.com/grpc/grpc/issues/18262. # If you are not encountering the error in above issue # go ahead and upload the docker image for new releases. @@ -287,6 +290,7 @@ LANG_RELEASE_MATRIX = { # v1.19 and v1.20 were deliberately omitted here because of an issue. # See https://github.com/grpc/grpc/issues/18264 ('v1.21.4', ReleaseInfo()), + ('v1.22.0', ReleaseInfo()), ]), 'csharp': OrderedDict([ @@ -317,5 +321,6 @@ LANG_RELEASE_MATRIX = { ('v1.19.0', ReleaseInfo(testcases_file='csharp__v1.18.0')), ('v1.20.0', ReleaseInfo()), ('v1.21.4', ReleaseInfo()), + ('v1.22.0', ReleaseInfo()), ]), } From bd5ed4fddd7e3aa9121a9eed92b820ef37e03b48 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 24 Jun 2019 16:37:08 -0700 Subject: [PATCH 34/55] Move compiler targets from /BUILD to /src/compiler/BUILD --- BUILD | 87 ------------- BUILD.gn | 1 + bazel/cc_grpc_library.bzl | 2 +- bazel/grpc_deps.bzl | 2 +- bazel/python_rules.bzl | 2 +- build.yaml | 1 + src/compiler/BUILD | 120 ++++++++++++++++++ src/compiler/config.h | 30 +---- src/compiler/config_protobuf.h | 52 ++++++++ .../generated/sources_and_headers.json | 2 + 10 files changed, 180 insertions(+), 119 deletions(-) create mode 100644 src/compiler/BUILD create mode 100644 src/compiler/config_protobuf.h diff --git a/BUILD b/BUILD index 95d8f753e39..b5d7761a9a1 100644 --- a/BUILD +++ b/BUILD @@ -30,7 +30,6 @@ load( "//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_generate_one_off_targets", - "grpc_proto_plugin", ) config_setting( @@ -429,92 +428,6 @@ grpc_cc_library( ], ) -grpc_cc_library( - name = "grpc_plugin_support", - srcs = [ - "src/compiler/cpp_generator.cc", - "src/compiler/csharp_generator.cc", - "src/compiler/node_generator.cc", - "src/compiler/objective_c_generator.cc", - "src/compiler/php_generator.cc", - "src/compiler/python_generator.cc", - "src/compiler/ruby_generator.cc", - ], - hdrs = [ - "src/compiler/config.h", - "src/compiler/cpp_generator.h", - "src/compiler/cpp_generator_helpers.h", - "src/compiler/cpp_plugin.h", - "src/compiler/csharp_generator.h", - "src/compiler/csharp_generator_helpers.h", - "src/compiler/generator_helpers.h", - "src/compiler/node_generator.h", - "src/compiler/node_generator_helpers.h", - "src/compiler/objective_c_generator.h", - "src/compiler/objective_c_generator_helpers.h", - "src/compiler/php_generator.h", - "src/compiler/php_generator_helpers.h", - "src/compiler/protobuf_plugin.h", - "src/compiler/python_generator.h", - "src/compiler/python_generator_helpers.h", - "src/compiler/python_private_generator.h", - "src/compiler/ruby_generator.h", - "src/compiler/ruby_generator_helpers-inl.h", - "src/compiler/ruby_generator_map-inl.h", - "src/compiler/ruby_generator_string-inl.h", - "src/compiler/schema_interface.h", - ], - external_deps = [ - "protobuf_clib", - ], - language = "c++", - deps = [ - "grpc++_config_proto", - ], -) - -grpc_proto_plugin( - name = "grpc_cpp_plugin", - srcs = ["src/compiler/cpp_plugin.cc"], - deps = [":grpc_plugin_support"], -) - -grpc_proto_plugin( - name = "grpc_csharp_plugin", - srcs = ["src/compiler/csharp_plugin.cc"], - deps = [":grpc_plugin_support"], -) - -grpc_proto_plugin( - name = "grpc_node_plugin", - srcs = ["src/compiler/node_plugin.cc"], - deps = [":grpc_plugin_support"], -) - -grpc_proto_plugin( - name = "grpc_objective_c_plugin", - srcs = ["src/compiler/objective_c_plugin.cc"], - deps = [":grpc_plugin_support"], -) - -grpc_proto_plugin( - name = "grpc_php_plugin", - srcs = ["src/compiler/php_plugin.cc"], - deps = [":grpc_plugin_support"], -) - -grpc_proto_plugin( - name = "grpc_python_plugin", - srcs = ["src/compiler/python_plugin.cc"], - deps = [":grpc_plugin_support"], -) - -grpc_proto_plugin( - name = "grpc_ruby_plugin", - srcs = ["src/compiler/ruby_plugin.cc"], - deps = [":grpc_plugin_support"], -) - grpc_cc_library( name = "grpc_csharp_ext", srcs = [ diff --git a/BUILD.gn b/BUILD.gn index 0868b2dae5a..177f4082adf 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -1413,6 +1413,7 @@ config("grpc_config") { "include/grpc++/impl/codegen/config_protobuf.h", "include/grpcpp/impl/codegen/config_protobuf.h", "src/compiler/config.h", + "src/compiler/config_protobuf.h", "src/compiler/cpp_generator.cc", "src/compiler/cpp_generator.h", "src/compiler/cpp_generator_helpers.h", diff --git a/bazel/cc_grpc_library.bzl b/bazel/cc_grpc_library.bzl index 572af756176..dea493eaf20 100644 --- a/bazel/cc_grpc_library.bzl +++ b/bazel/cc_grpc_library.bzl @@ -88,7 +88,7 @@ def cc_grpc_library( generate_cc( name = codegen_grpc_target, srcs = proto_targets, - plugin = "@com_github_grpc_grpc//:grpc_cpp_plugin", + plugin = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin", well_known_protos = well_known_protos, generate_mocks = generate_mocks, **kwargs diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index be58c52b2b9..49def4f82c1 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -78,7 +78,7 @@ def grpc_deps(): native.bind( name = "grpc_cpp_plugin", - actual = "@com_github_grpc_grpc//:grpc_cpp_plugin", + actual = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin", ) native.bind( diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl index d4ff77094cc..14550852a4a 100644 --- a/bazel/python_rules.bzl +++ b/bazel/python_rules.bzl @@ -163,7 +163,7 @@ def py_proto_library( _generate_py( name = codegen_grpc_target, deps = deps, - plugin = "//:grpc_python_plugin", + plugin = "//src/compiler:grpc_python_plugin", well_known_protos = well_known_protos, **kwargs ) diff --git a/build.yaml b/build.yaml index 4134ecd8880..5915b9ccd64 100644 --- a/build.yaml +++ b/build.yaml @@ -1969,6 +1969,7 @@ libs: language: c++ headers: - src/compiler/config.h + - src/compiler/config_protobuf.h - src/compiler/cpp_generator.h - src/compiler/cpp_generator_helpers.h - src/compiler/csharp_generator.h diff --git a/src/compiler/BUILD b/src/compiler/BUILD new file mode 100644 index 00000000000..9a38f942939 --- /dev/null +++ b/src/compiler/BUILD @@ -0,0 +1,120 @@ +# gRPC Bazel BUILD file. +# +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache v2 + +exports_files(["LICENSE"]) + +package( + default_visibility = ["//visibility:public"], + features = [ + "-layering_check", + "-parse_headers", + ], +) + +load( + "//bazel:grpc_build_system.bzl", + "grpc_cc_library", + "grpc_proto_plugin", +) + +grpc_cc_library( + name = "grpc_plugin_support", + srcs = [ + "cpp_generator.cc", + "csharp_generator.cc", + "node_generator.cc", + "objective_c_generator.cc", + "php_generator.cc", + "python_generator.cc", + "ruby_generator.cc", + ], + hdrs = [ + "config_protobuf.h", + "config.h", + "cpp_generator.h", + "cpp_generator_helpers.h", + "cpp_plugin.h", + "csharp_generator.h", + "csharp_generator_helpers.h", + "generator_helpers.h", + "node_generator.h", + "node_generator_helpers.h", + "objective_c_generator.h", + "objective_c_generator_helpers.h", + "php_generator.h", + "php_generator_helpers.h", + "protobuf_plugin.h", + "python_generator.h", + "python_generator_helpers.h", + "python_private_generator.h", + "ruby_generator.h", + "ruby_generator_helpers-inl.h", + "ruby_generator_map-inl.h", + "ruby_generator_string-inl.h", + "schema_interface.h", + ], + external_deps = [ + "protobuf_clib", + ], + language = "c++", + deps = [ + "//:grpc++_config_proto", + ], +) + +grpc_proto_plugin( + name = "grpc_cpp_plugin", + srcs = ["cpp_plugin.cc"], + deps = [":grpc_plugin_support"], +) + +grpc_proto_plugin( + name = "grpc_csharp_plugin", + srcs = ["csharp_plugin.cc"], + deps = [":grpc_plugin_support"], +) + +grpc_proto_plugin( + name = "grpc_node_plugin", + srcs = ["node_plugin.cc"], + deps = [":grpc_plugin_support"], +) + +grpc_proto_plugin( + name = "grpc_objective_c_plugin", + srcs = ["objective_c_plugin.cc"], + deps = [":grpc_plugin_support"], +) + +grpc_proto_plugin( + name = "grpc_php_plugin", + srcs = ["php_plugin.cc"], + deps = [":grpc_plugin_support"], +) + +grpc_proto_plugin( + name = "grpc_python_plugin", + srcs = ["python_plugin.cc"], + deps = [":grpc_plugin_support"], +) + +grpc_proto_plugin( + name = "grpc_ruby_plugin", + srcs = ["ruby_plugin.cc"], + deps = [":grpc_plugin_support"], +) diff --git a/src/compiler/config.h b/src/compiler/config.h index cfdc3036247..60772f2be68 100644 --- a/src/compiler/config.h +++ b/src/compiler/config.h @@ -19,35 +19,7 @@ #ifndef SRC_COMPILER_CONFIG_H #define SRC_COMPILER_CONFIG_H -#include - -#ifndef GRPC_CUSTOM_CODEGENERATOR -#include -#define GRPC_CUSTOM_CODEGENERATOR ::google::protobuf::compiler::CodeGenerator -#define GRPC_CUSTOM_GENERATORCONTEXT \ - ::google::protobuf::compiler::GeneratorContext -#endif - -#ifndef GRPC_CUSTOM_PRINTER -#include -#include -#include -#define GRPC_CUSTOM_PRINTER ::google::protobuf::io::Printer -#define GRPC_CUSTOM_CODEDOUTPUTSTREAM ::google::protobuf::io::CodedOutputStream -#define GRPC_CUSTOM_STRINGOUTPUTSTREAM \ - ::google::protobuf::io::StringOutputStream -#endif - -#ifndef GRPC_CUSTOM_PLUGINMAIN -#include -#define GRPC_CUSTOM_PLUGINMAIN ::google::protobuf::compiler::PluginMain -#endif - -#ifndef GRPC_CUSTOM_PARSEGENERATORPARAMETER -#include -#define GRPC_CUSTOM_PARSEGENERATORPARAMETER \ - ::google::protobuf::compiler::ParseGeneratorParameter -#endif +#include "src/compiler/config_protobuf.h" #ifndef GRPC_CUSTOM_STRING #include diff --git a/src/compiler/config_protobuf.h b/src/compiler/config_protobuf.h new file mode 100644 index 00000000000..06d5073f433 --- /dev/null +++ b/src/compiler/config_protobuf.h @@ -0,0 +1,52 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef SRC_COMPILER_CONFIG_PROTOBUF_H +#define SRC_COMPILER_CONFIG_PROTOBUF_H + +#include + +#ifndef GRPC_CUSTOM_CODEGENERATOR +#include +#define GRPC_CUSTOM_CODEGENERATOR ::google::protobuf::compiler::CodeGenerator +#define GRPC_CUSTOM_GENERATORCONTEXT \ + ::google::protobuf::compiler::GeneratorContext +#endif + +#ifndef GRPC_CUSTOM_PRINTER +#include +#include +#include +#define GRPC_CUSTOM_PRINTER ::google::protobuf::io::Printer +#define GRPC_CUSTOM_CODEDOUTPUTSTREAM ::google::protobuf::io::CodedOutputStream +#define GRPC_CUSTOM_STRINGOUTPUTSTREAM \ + ::google::protobuf::io::StringOutputStream +#endif + +#ifndef GRPC_CUSTOM_PLUGINMAIN +#include +#define GRPC_CUSTOM_PLUGINMAIN ::google::protobuf::compiler::PluginMain +#endif + +#ifndef GRPC_CUSTOM_PARSEGENERATORPARAMETER +#include +#define GRPC_CUSTOM_PARSEGENERATORPARAMETER \ + ::google::protobuf::compiler::ParseGeneratorParameter +#endif + +#endif // SRC_COMPILER_CONFIG_PROTOBUF_H diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 0f560fd1063..7a1baae36cd 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7184,6 +7184,7 @@ ], "headers": [ "src/compiler/config.h", + "src/compiler/config_protobuf.h", "src/compiler/cpp_generator.h", "src/compiler/cpp_generator_helpers.h", "src/compiler/csharp_generator.h", @@ -7210,6 +7211,7 @@ "name": "grpc_plugin_support", "src": [ "src/compiler/config.h", + "src/compiler/config_protobuf.h", "src/compiler/cpp_generator.cc", "src/compiler/cpp_generator.h", "src/compiler/cpp_generator_helpers.h", From 5779dd935a22c699a1c33be35b32bee2053a3aba Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 10 Jul 2019 13:52:31 -0700 Subject: [PATCH 35/55] Qualify the error code with StatusCode:: --- src/cpp/client/secure_credentials.cc | 21 ++++++++++++--------- test/cpp/client/credentials_test.cc | 10 +++++----- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc index 5de5a76194e..ebff8af3e5a 100644 --- a/src/cpp/client/secure_credentials.cc +++ b/src/cpp/client/secure_credentials.cc @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include @@ -139,7 +139,8 @@ grpc::Status StsCredentialsOptionsFromJson(const grpc::string& json_string, void operator()(grpc_json* json) { grpc_json_destroy(json); } }; if (options == nullptr) { - return grpc::Status(grpc::INVALID_ARGUMENT, "options cannot be nullptr."); + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + "options cannot be nullptr."); } ClearStsCredentialsOptions(options); std::vector scratchpad(json_string.c_str(), @@ -147,7 +148,7 @@ grpc::Status StsCredentialsOptionsFromJson(const grpc::string& json_string, std::unique_ptr json( grpc_json_parse_string(&scratchpad[0])); if (json == nullptr) { - return grpc::Status(grpc::INVALID_ARGUMENT, "Invalid json."); + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid json."); } // Required fields. @@ -155,7 +156,7 @@ grpc::Status StsCredentialsOptionsFromJson(const grpc::string& json_string, json.get(), "token_exchange_service_uri", nullptr); if (value == nullptr) { ClearStsCredentialsOptions(options); - return grpc::Status(grpc::INVALID_ARGUMENT, + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "token_exchange_service_uri must be specified."); } options->token_exchange_service_uri.assign(value); @@ -163,7 +164,7 @@ grpc::Status StsCredentialsOptionsFromJson(const grpc::string& json_string, grpc_json_get_string_property(json.get(), "subject_token_path", nullptr); if (value == nullptr) { ClearStsCredentialsOptions(options); - return grpc::Status(grpc::INVALID_ARGUMENT, + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "subject_token_path must be specified."); } options->subject_token_path.assign(value); @@ -171,7 +172,7 @@ grpc::Status StsCredentialsOptionsFromJson(const grpc::string& json_string, grpc_json_get_string_property(json.get(), "subject_token_type", nullptr); if (value == nullptr) { ClearStsCredentialsOptions(options); - return grpc::Status(grpc::INVALID_ARGUMENT, + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "subject_token_type must be specified."); } options->subject_token_type.assign(value); @@ -199,7 +200,8 @@ grpc::Status StsCredentialsOptionsFromJson(const grpc::string& json_string, // Builds STS credentials Options from the $STS_CREDENTIALS env var. grpc::Status StsCredentialsOptionsFromEnv(StsCredentialsOptions* options) { if (options == nullptr) { - return grpc::Status(grpc::INVALID_ARGUMENT, "options cannot be nullptr."); + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, + "options cannot be nullptr."); } ClearStsCredentialsOptions(options); grpc_slice json_string = grpc_empty_slice(); @@ -214,13 +216,14 @@ grpc::Status StsCredentialsOptionsFromEnv(StsCredentialsOptions* options) { }; if (sts_creds_path == nullptr) { - status = grpc::Status(grpc::NOT_FOUND, + status = grpc::Status(grpc::StatusCode::NOT_FOUND, "STS_CREDENTIALS environment variable not set."); return cleanup(); } error = grpc_load_file(sts_creds_path, 1, &json_string); if (error != GRPC_ERROR_NONE) { - status = grpc::Status(grpc::NOT_FOUND, grpc_error_string(error)); + status = + grpc::Status(grpc::StatusCode::NOT_FOUND, grpc_error_string(error)); return cleanup(); } status = StsCredentialsOptionsFromJson( diff --git a/test/cpp/client/credentials_test.cc b/test/cpp/client/credentials_test.cc index d560fb6d8e8..ba004efe0d9 100644 --- a/test/cpp/client/credentials_test.cc +++ b/test/cpp/client/credentials_test.cc @@ -119,7 +119,7 @@ TEST_F(CredentialsTest, StsCredentialsOptionsJson) { I'm not a valid JSON. )"; EXPECT_EQ( - grpc::INVALID_ARGUMENT, + grpc::StatusCode::INVALID_ARGUMENT, grpc::experimental::StsCredentialsOptionsFromJson(invalid_json, &options) .error_code()); @@ -130,7 +130,7 @@ TEST_F(CredentialsTest, StsCredentialsOptionsJson) { })"; auto status = grpc::experimental::StsCredentialsOptionsFromJson( invalid_json_missing_subject_token_type, &options); - EXPECT_EQ(grpc::INVALID_ARGUMENT, status.error_code()); + EXPECT_EQ(grpc::StatusCode::INVALID_ARGUMENT, status.error_code()); EXPECT_THAT(status.error_message(), ::testing::HasSubstr("subject_token_type")); @@ -141,7 +141,7 @@ TEST_F(CredentialsTest, StsCredentialsOptionsJson) { })"; status = grpc::experimental::StsCredentialsOptionsFromJson( invalid_json_missing_subject_token_path, &options); - EXPECT_EQ(grpc::INVALID_ARGUMENT, status.error_code()); + EXPECT_EQ(grpc::StatusCode::INVALID_ARGUMENT, status.error_code()); EXPECT_THAT(status.error_message(), ::testing::HasSubstr("subject_token_path")); @@ -152,7 +152,7 @@ TEST_F(CredentialsTest, StsCredentialsOptionsJson) { })"; status = grpc::experimental::StsCredentialsOptionsFromJson( invalid_json_missing_token_exchange_uri, &options); - EXPECT_EQ(grpc::INVALID_ARGUMENT, status.error_code()); + EXPECT_EQ(grpc::StatusCode::INVALID_ARGUMENT, status.error_code()); EXPECT_THAT(status.error_message(), ::testing::HasSubstr("token_exchange_service_uri")); } @@ -162,7 +162,7 @@ TEST_F(CredentialsTest, StsCredentialsOptionsFromEnv) { gpr_unsetenv("STS_CREDENTIALS"); grpc::experimental::StsCredentialsOptions options; auto status = grpc::experimental::StsCredentialsOptionsFromEnv(&options); - EXPECT_EQ(grpc::NOT_FOUND, status.error_code()); + EXPECT_EQ(grpc::StatusCode::NOT_FOUND, status.error_code()); // Set env and check for success. const char valid_json[] = R"( From a2eb267ccb7440212e21cbbd405e712fe9db78ea Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Wed, 10 Jul 2019 14:29:55 -0700 Subject: [PATCH 36/55] Clarify std:: usage --- doc/core/moving-to-c++.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/core/moving-to-c++.md b/doc/core/moving-to-c++.md index 4c745b38a90..717e2267b33 100644 --- a/doc/core/moving-to-c++.md +++ b/doc/core/moving-to-c++.md @@ -21,9 +21,13 @@ C++ compatible with ## Constraints -- No use of standard library +- No use of standard library if it requires link-time dependency - Standard library makes wrapping difficult/impossible and also reduces platform portability - This takes precedence over using C++ style guide +- Limited use of standard library if it does not require link-time dependency + - We can use things from `std::` as long as they are header-only implementations. + - Since the standard library API does not specify whether any given part of the API is implemented header-only, the only way to know is to try using something and see if our tests fail. + - Since there is no guarantee that some header-only implementation in the standard library will remain header-only in the future, we should define our own API in lib/gprpp that is an alias for the thing we want to use in `std::` and use the gprpp API in core. That way, if we later need to stop using the thing from `std::`, we can replace the alias with our own implementation. - But lambdas are ok - As are third-party libraries that meet our build requirements (such as many parts of abseil) - There will be some C++ features that don't work From 7e3a9b6b236174000e673771cfbae22a42d05621 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Wed, 10 Jul 2019 14:54:44 -0700 Subject: [PATCH 37/55] Remove error arg - reviewer comment --- .../transport/chttp2/transport/chttp2_transport.cc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index b05a48a6a0c..6254369ef69 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1232,10 +1232,10 @@ static grpc_closure* add_closure_barrier(grpc_closure* closure) { return closure; } -static void null_then_sched_closure(grpc_closure** closure, grpc_error* error) { +static void null_then_sched_closure(grpc_closure** closure) { grpc_closure* c = *closure; *closure = nullptr; - GRPC_CLOSURE_SCHED(c, error); + GRPC_CLOSURE_SCHED(c, GRPC_ERROR_NONE); } void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t, @@ -1902,7 +1902,7 @@ void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_chttp2_transport* t, } grpc_chttp2_incoming_metadata_buffer_publish(&s->metadata_buffer[0], s->recv_initial_metadata); - null_then_sched_closure(&s->recv_initial_metadata_ready, GRPC_ERROR_NONE); + null_then_sched_closure(&s->recv_initial_metadata_ready); } } @@ -1982,10 +1982,10 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_chttp2_transport* t, s->unprocessed_incoming_frames_buffer_cached_length = s->unprocessed_incoming_frames_buffer.length; if (error == GRPC_ERROR_NONE && *s->recv_message != nullptr) { - null_then_sched_closure(&s->recv_message_ready, GRPC_ERROR_NONE); + null_then_sched_closure(&s->recv_message_ready); } else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) { *s->recv_message = nullptr; - null_then_sched_closure(&s->recv_message_ready, GRPC_ERROR_NONE); + null_then_sched_closure(&s->recv_message_ready); } GRPC_ERROR_UNREF(error); } @@ -2051,8 +2051,7 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_chttp2_transport* t, s->collecting_stats = nullptr; grpc_chttp2_incoming_metadata_buffer_publish(&s->metadata_buffer[1], s->recv_trailing_metadata); - null_then_sched_closure(&s->recv_trailing_metadata_finished, - GRPC_ERROR_NONE); + null_then_sched_closure(&s->recv_trailing_metadata_finished); } } } From 444806583d7aace199b3aa83051d08315239fa7e Mon Sep 17 00:00:00 2001 From: Qiancheng Zhao Date: Wed, 10 Jul 2019 15:26:53 -0700 Subject: [PATCH 38/55] lazy resolving lb policy creation --- .../filters/client_channel/client_channel.cc | 100 ++++++++++-------- .../resolver/fake/fake_resolver.cc | 1 - .../client_channel/resolving_lb_policy.cc | 59 ++--------- .../client_channel/resolving_lb_policy.h | 15 +-- test/cpp/end2end/grpclb_end2end_test.cc | 4 + 5 files changed, 73 insertions(+), 106 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 0b612e67a33..39ca3e5abb2 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -210,6 +210,10 @@ class ChannelData { ChannelData(grpc_channel_element_args* args, grpc_error** error); ~ChannelData(); + void CreateResolvingLoadBalancingPolicyLocked(); + + void DestroyResolvingLoadBalancingPolicyLocked(); + static bool ProcessResolverResultLocked( void* arg, const Resolver::Result& result, const char** lb_policy_name, RefCountedPtr* lb_policy_config, @@ -235,8 +239,10 @@ class ChannelData { const size_t per_rpc_retry_buffer_size_; grpc_channel_stack* owning_stack_; ClientChannelFactory* client_channel_factory_; - UniquePtr server_name_; + const grpc_channel_args* channel_args_; RefCountedPtr default_service_config_; + UniquePtr server_name_; + UniquePtr target_uri_; channelz::ChannelNode* channelz_node_; // @@ -1226,59 +1232,61 @@ ChannelData::ChannelData(grpc_channel_element_args* args, grpc_error** error) grpc_channel_args* new_args = nullptr; grpc_proxy_mappers_map_name(server_uri, args->channel_args, &proxy_name, &new_args); - UniquePtr target_uri(proxy_name != nullptr ? proxy_name - : gpr_strdup(server_uri)); + target_uri_.reset(proxy_name != nullptr ? proxy_name + : gpr_strdup(server_uri)); + channel_args_ = new_args != nullptr + ? new_args + : grpc_channel_args_copy(args->channel_args); + if (!ResolverRegistry::IsValidTarget(target_uri_.get())) { + *error = + GRPC_ERROR_CREATE_FROM_STATIC_STRING("the target uri is not valid."); + return; + } + *error = GRPC_ERROR_NONE; +} + +ChannelData::~ChannelData() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { + gpr_log(GPR_INFO, "chand=%p: destroying channel", this); + } + DestroyResolvingLoadBalancingPolicyLocked(); + grpc_channel_args_destroy(channel_args_); + // Stop backup polling. + grpc_client_channel_stop_backup_polling(interested_parties_); + grpc_pollset_set_destroy(interested_parties_); + GRPC_COMBINER_UNREF(data_plane_combiner_, "client_channel"); + GRPC_COMBINER_UNREF(combiner_, "client_channel"); + GRPC_ERROR_UNREF(disconnect_error_.Load(MemoryOrder::RELAXED)); + grpc_connectivity_state_destroy(&state_tracker_); + gpr_mu_destroy(&info_mu_); +} + +void ChannelData::CreateResolvingLoadBalancingPolicyLocked() { // Instantiate resolving LB policy. LoadBalancingPolicy::Args lb_args; lb_args.combiner = combiner_; lb_args.channel_control_helper = UniquePtr( New(this)); - lb_args.args = new_args != nullptr ? new_args : args->channel_args; + lb_args.args = channel_args_; + UniquePtr target_uri(strdup(target_uri_.get())); resolving_lb_policy_.reset(New( std::move(lb_args), &grpc_client_channel_routing_trace, - std::move(target_uri), ProcessResolverResultLocked, this, error)); - grpc_channel_args_destroy(new_args); - if (*error != GRPC_ERROR_NONE) { - // Orphan the resolving LB policy and flush the exec_ctx to ensure - // that it finishes shutting down. This ensures that if we are - // failing, we destroy the ClientChannelControlHelper (and thus - // unref the channel stack) before we return. - // TODO(roth): This is not a complete solution, because it only - // catches the case where channel stack initialization fails in this - // particular filter. If there is a failure in a different filter, we - // will leave a dangling ref here, which can cause a crash. Fortunately, - // in practice, there are no other filters that can cause failures in - // channel stack initialization, so this works for now. - resolving_lb_policy_.reset(); - ExecCtx::Get()->Flush(); - } else { - grpc_pollset_set_add_pollset_set(resolving_lb_policy_->interested_parties(), - interested_parties_); - if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { - gpr_log(GPR_INFO, "chand=%p: created resolving_lb_policy=%p", this, - resolving_lb_policy_.get()); - } + std::move(target_uri), ProcessResolverResultLocked, this)); + grpc_pollset_set_add_pollset_set(resolving_lb_policy_->interested_parties(), + interested_parties_); + if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { + gpr_log(GPR_INFO, "chand=%p: created resolving_lb_policy=%p", this, + resolving_lb_policy_.get()); } } -ChannelData::~ChannelData() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) { - gpr_log(GPR_INFO, "chand=%p: destroying channel", this); - } +void ChannelData::DestroyResolvingLoadBalancingPolicyLocked() { if (resolving_lb_policy_ != nullptr) { grpc_pollset_set_del_pollset_set(resolving_lb_policy_->interested_parties(), interested_parties_); resolving_lb_policy_.reset(); } - // Stop backup polling. - grpc_client_channel_stop_backup_polling(interested_parties_); - grpc_pollset_set_destroy(interested_parties_); - GRPC_COMBINER_UNREF(data_plane_combiner_, "client_channel"); - GRPC_COMBINER_UNREF(combiner_, "client_channel"); - GRPC_ERROR_UNREF(disconnect_error_.Load(MemoryOrder::RELAXED)); - grpc_connectivity_state_destroy(&state_tracker_); - gpr_mu_destroy(&info_mu_); } void ChannelData::ProcessLbPolicy( @@ -1500,10 +1508,7 @@ void ChannelData::StartTransportOpLocked(void* arg, grpc_error* ignored) { GPR_ASSERT(chand->disconnect_error_.CompareExchangeStrong( &error, op->disconnect_with_error, MemoryOrder::ACQ_REL, MemoryOrder::ACQUIRE)); - grpc_pollset_set_del_pollset_set( - chand->resolving_lb_policy_->interested_parties(), - chand->interested_parties_); - chand->resolving_lb_policy_.reset(); + chand->DestroyResolvingLoadBalancingPolicyLocked(); // Will delete itself. New( chand, GRPC_CHANNEL_SHUTDOWN, "shutdown from API", @@ -1574,6 +1579,8 @@ void ChannelData::TryToConnectLocked(void* arg, grpc_error* error_ignored) { auto* chand = static_cast(arg); if (chand->resolving_lb_policy_ != nullptr) { chand->resolving_lb_policy_->ExitIdleLocked(); + } else { + chand->CreateResolvingLoadBalancingPolicyLocked(); } GRPC_CHANNEL_STACK_UNREF(chand->owning_stack_, "TryToConnect"); } @@ -3463,6 +3470,15 @@ void CallData::StartPickLocked(void* arg, grpc_error* error) { ChannelData* chand = static_cast(elem->channel_data); GPR_ASSERT(calld->connected_subchannel_ == nullptr); GPR_ASSERT(calld->subchannel_call_ == nullptr); + // picker's being null means the channel is currently in IDLE state. The + // incoming call will make the channel exit IDLE and queue itself. + if (chand->picker() == nullptr) { + // We are currently in the data plane. + // Bounce into the control plane to exit IDLE. + chand->CheckConnectivityState(true); + calld->AddCallToQueuedPicksLocked(elem); + return; + } // Apply service config to call if needed. calld->MaybeApplyServiceConfigToCallLocked(elem); // If this is a retry, use the send_initial_metadata payload that diff --git a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc index c5a27f127b8..761b27292da 100644 --- a/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc @@ -196,7 +196,6 @@ void FakeResolverResponseGenerator::SetResponse(Resolver::Result result) { grpc_combiner_scheduler(resolver_->combiner())), GRPC_ERROR_NONE); } else { - GPR_ASSERT(!has_result_); has_result_ = true; result_ = std::move(result); } diff --git a/src/core/ext/filters/client_channel/resolving_lb_policy.cc b/src/core/ext/filters/client_channel/resolving_lb_policy.cc index 180b0fcbbf4..4b61df09959 100644 --- a/src/core/ext/filters/client_channel/resolving_lb_policy.cc +++ b/src/core/ext/filters/client_channel/resolving_lb_policy.cc @@ -181,52 +181,29 @@ class ResolvingLoadBalancingPolicy::ResolvingControlHelper // ResolvingLoadBalancingPolicy // -ResolvingLoadBalancingPolicy::ResolvingLoadBalancingPolicy( - Args args, TraceFlag* tracer, UniquePtr target_uri, - UniquePtr child_policy_name, - RefCountedPtr child_lb_config, - grpc_error** error) - : LoadBalancingPolicy(std::move(args)), - tracer_(tracer), - target_uri_(std::move(target_uri)), - child_policy_name_(std::move(child_policy_name)), - child_lb_config_(std::move(child_lb_config)) { - GPR_ASSERT(child_policy_name_ != nullptr); - // Don't fetch service config, since this ctor is for use in nested LB - // policies, not at the top level, and we only fetch the service - // config at the top level. - grpc_arg arg = grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION), 0); - grpc_channel_args* new_args = - grpc_channel_args_copy_and_add(args.args, &arg, 1); - *error = Init(*new_args); - grpc_channel_args_destroy(new_args); -} - ResolvingLoadBalancingPolicy::ResolvingLoadBalancingPolicy( Args args, TraceFlag* tracer, UniquePtr target_uri, ProcessResolverResultCallback process_resolver_result, - void* process_resolver_result_user_data, grpc_error** error) + void* process_resolver_result_user_data) : LoadBalancingPolicy(std::move(args)), tracer_(tracer), target_uri_(std::move(target_uri)), process_resolver_result_(process_resolver_result), process_resolver_result_user_data_(process_resolver_result_user_data) { GPR_ASSERT(process_resolver_result != nullptr); - *error = Init(*args.args); -} - -grpc_error* ResolvingLoadBalancingPolicy::Init(const grpc_channel_args& args) { resolver_ = ResolverRegistry::CreateResolver( - target_uri_.get(), &args, interested_parties(), combiner(), + target_uri_.get(), args.args, interested_parties(), combiner(), UniquePtr(New(Ref()))); - if (resolver_ == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("resolver creation failed"); + // Since the validity of args has been checked when create the channel, + // CreateResolver() must return a non-null result. + GPR_ASSERT(resolver_ != nullptr); + if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) { + gpr_log(GPR_INFO, "resolving_lb=%p: starting name resolution", this); } - // Return our picker to the channel. channel_control_helper()->UpdateState( - GRPC_CHANNEL_IDLE, UniquePtr(New(Ref()))); - return GRPC_ERROR_NONE; + GRPC_CHANNEL_CONNECTING, + UniquePtr(New(Ref()))); + resolver_->StartLocked(); } ResolvingLoadBalancingPolicy::~ResolvingLoadBalancingPolicy() { @@ -262,10 +239,6 @@ void ResolvingLoadBalancingPolicy::ExitIdleLocked() { if (lb_policy_ != nullptr) { lb_policy_->ExitIdleLocked(); if (pending_lb_policy_ != nullptr) pending_lb_policy_->ExitIdleLocked(); - } else { - if (!started_resolving_ && resolver_ != nullptr) { - StartResolvingLocked(); - } } } @@ -278,18 +251,6 @@ void ResolvingLoadBalancingPolicy::ResetBackoffLocked() { if (pending_lb_policy_ != nullptr) pending_lb_policy_->ResetBackoffLocked(); } -void ResolvingLoadBalancingPolicy::StartResolvingLocked() { - if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) { - gpr_log(GPR_INFO, "resolving_lb=%p: starting name resolution", this); - } - GPR_ASSERT(!started_resolving_); - started_resolving_ = true; - channel_control_helper()->UpdateState( - GRPC_CHANNEL_CONNECTING, - UniquePtr(New(Ref()))); - resolver_->StartLocked(); -} - void ResolvingLoadBalancingPolicy::OnResolverError(grpc_error* error) { if (resolver_ == nullptr) { GRPC_ERROR_UNREF(error); diff --git a/src/core/ext/filters/client_channel/resolving_lb_policy.h b/src/core/ext/filters/client_channel/resolving_lb_policy.h index e91ea832661..d8447c18cf5 100644 --- a/src/core/ext/filters/client_channel/resolving_lb_policy.h +++ b/src/core/ext/filters/client_channel/resolving_lb_policy.h @@ -51,16 +51,6 @@ namespace grpc_core { // child LB policy and config to use. class ResolvingLoadBalancingPolicy : public LoadBalancingPolicy { public: - // If error is set when this returns, then construction failed, and - // the caller may not use the new object. - ResolvingLoadBalancingPolicy( - Args args, TraceFlag* tracer, UniquePtr target_uri, - UniquePtr child_policy_name, - RefCountedPtr child_lb_config, - grpc_error** error); - - // Private ctor, to be used by client_channel only! - // // Synchronous callback that takes the resolver result and sets // lb_policy_name and lb_policy_config to point to the right data. // Returns true if the service config has changed since the last result. @@ -77,7 +67,7 @@ class ResolvingLoadBalancingPolicy : public LoadBalancingPolicy { ResolvingLoadBalancingPolicy( Args args, TraceFlag* tracer, UniquePtr target_uri, ProcessResolverResultCallback process_resolver_result, - void* process_resolver_result_user_data, grpc_error** error); + void* process_resolver_result_user_data); virtual const char* name() const override { return "resolving_lb"; } @@ -98,10 +88,8 @@ class ResolvingLoadBalancingPolicy : public LoadBalancingPolicy { ~ResolvingLoadBalancingPolicy(); - grpc_error* Init(const grpc_channel_args& args); void ShutdownLocked() override; - void StartResolvingLocked(); void OnResolverError(grpc_error* error); void CreateOrUpdateLbPolicyLocked( const char* lb_policy_name, @@ -126,7 +114,6 @@ class ResolvingLoadBalancingPolicy : public LoadBalancingPolicy { // Resolver and associated state. OrphanablePtr resolver_; - bool started_resolving_ = false; bool previous_resolution_contained_addresses_ = false; // Child LB policy. diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index e3b79c810cd..d71dafc855f 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -1618,6 +1618,8 @@ TEST_F(UpdatesTest, ReresolveDeadBackend) { addresses.emplace_back(AddressData{balancers_[0]->port_, true, ""}); addresses.emplace_back(AddressData{backends_[0]->port_, false, ""}); SetNextResolution(addresses); + // Ask channel to connect to trigger resolver creation. + channel_->GetState(true); // The re-resolution result will contain the addresses of the same balancer // and a new fallback backend. addresses.clear(); @@ -1669,6 +1671,8 @@ class UpdatesWithClientLoadReportingTest : public GrpclbEnd2endTest { }; TEST_F(UpdatesWithClientLoadReportingTest, ReresolveDeadBalancer) { + // Ask channel to connect to trigger resolver creation. + channel_->GetState(true); std::vector addresses; addresses.emplace_back(AddressData{balancers_[0]->port_, true, ""}); SetNextResolution(addresses); From 39775cf30fe2b29693652184e01a480a8596a89c Mon Sep 17 00:00:00 2001 From: mgravell Date: Thu, 11 Jul 2019 13:07:50 +0100 Subject: [PATCH 39/55] remove delegate allocation and boxing from cancellation registrations --- src/csharp/Grpc.Core.Tests/CallCancellationTest.cs | 11 +++++++++++ src/csharp/Grpc.Core/Internal/AsyncCall.cs | 10 +++------- src/csharp/Grpc.Core/Internal/AsyncCallBase.cs | 8 ++++++++ src/csharp/Grpc.Core/Internal/ClientResponseStream.cs | 3 +-- src/csharp/Grpc.Core/Internal/ServerRequestStream.cs | 4 +--- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/CallCancellationTest.cs b/src/csharp/Grpc.Core.Tests/CallCancellationTest.cs index e040f52380a..753b8b227b1 100644 --- a/src/csharp/Grpc.Core.Tests/CallCancellationTest.cs +++ b/src/csharp/Grpc.Core.Tests/CallCancellationTest.cs @@ -178,5 +178,16 @@ namespace Grpc.Core.Tests Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode); } } + + [Test] + public void CanDisposeDefaultCancellationRegistration() + { + // prove that we're fine to dispose default CancellationTokenRegistration + // values without boxing them to IDisposable for a null-check + var obj = default(CancellationTokenRegistration); + obj.Dispose(); + + using (obj) {} + } } } diff --git a/src/csharp/Grpc.Core/Internal/AsyncCall.cs b/src/csharp/Grpc.Core/Internal/AsyncCall.cs index a1c688140d1..ce3508d398e 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCall.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCall.cs @@ -38,7 +38,7 @@ namespace Grpc.Core.Internal bool registeredWithChannel; // Dispose of to de-register cancellation token registration - IDisposable cancellationTokenRegistration; + CancellationTokenRegistration cancellationTokenRegistration; // Completion of a pending unary response if not null. TaskCompletionSource unaryResponseTcs; @@ -409,7 +409,7 @@ namespace Grpc.Core.Internal // deadlock. // See https://github.com/grpc/grpc/issues/14777 // See https://github.com/dotnet/corefx/issues/14903 - cancellationTokenRegistration?.Dispose(); + cancellationTokenRegistration.Dispose(); } protected override bool IsClient @@ -509,11 +509,7 @@ namespace Grpc.Core.Internal // Make sure that once cancellationToken for this call is cancelled, Cancel() will be called. private void RegisterCancellationCallback() { - var token = details.Options.CancellationToken; - if (token.CanBeCanceled) - { - cancellationTokenRegistration = token.Register(() => this.Cancel()); - } + cancellationTokenRegistration = RegisterCancellationCallbackForToken(details.Options.CancellationToken); } /// diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs index 0b99aaf3e21..ac1f9066516 100644 --- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs +++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs @@ -394,5 +394,13 @@ namespace Grpc.Core.Internal { HandleReadFinished(success, receivedMessageReader); } + + internal CancellationTokenRegistration RegisterCancellationCallbackForToken(CancellationToken cancellationToken) + { + if (cancellationToken.CanBeCanceled) return cancellationToken.Register(CancelCallFromToken, this); + return default(CancellationTokenRegistration); + } + + private static readonly Action CancelCallFromToken = state => ((AsyncCallBase)state).Cancel(); } } diff --git a/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs b/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs index ab649ee7662..ff73b26dff4 100644 --- a/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs +++ b/src/csharp/Grpc.Core/Internal/ClientResponseStream.cs @@ -49,8 +49,7 @@ namespace Grpc.Core.Internal public async Task MoveNext(CancellationToken token) { - var cancellationTokenRegistration = token.CanBeCanceled ? token.Register(() => call.Cancel()) : (IDisposable) null; - using (cancellationTokenRegistration) + using (call.RegisterCancellationCallbackForToken(token)) { var result = await call.ReadMessageAsync().ConfigureAwait(false); this.current = result; diff --git a/src/csharp/Grpc.Core/Internal/ServerRequestStream.cs b/src/csharp/Grpc.Core/Internal/ServerRequestStream.cs index 058dddb7ebb..f3655139e47 100644 --- a/src/csharp/Grpc.Core/Internal/ServerRequestStream.cs +++ b/src/csharp/Grpc.Core/Internal/ServerRequestStream.cs @@ -49,9 +49,7 @@ namespace Grpc.Core.Internal public async Task MoveNext(CancellationToken token) { - - var cancellationTokenRegistration = token.CanBeCanceled ? token.Register(() => call.Cancel()) : (IDisposable) null; - using (cancellationTokenRegistration) + using (call.RegisterCancellationCallbackForToken(token)) { var result = await call.ReadMessageAsync().ConfigureAwait(false); this.current = result; From 8e4d14fe91053fb4202db948e62dc11bfad8d6e5 Mon Sep 17 00:00:00 2001 From: Wensheng Tang Date: Sun, 7 Jul 2019 02:47:20 +0800 Subject: [PATCH 40/55] Fix the assertion of grpc_udp_server_add_port grpc_udp_server_add_port() can return port = -1 when going fail. @nanahpang also recommands discarding port = 0 --- test/core/iomgr/udp_server_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/core/iomgr/udp_server_test.cc b/test/core/iomgr/udp_server_test.cc index f65783a0174..3c9e7108608 100644 --- a/test/core/iomgr/udp_server_test.cc +++ b/test/core/iomgr/udp_server_test.cc @@ -218,7 +218,7 @@ static void test_no_op_with_port(void) { addr->sin_family = AF_INET; GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size, snd_buf_size, &handler_factory, - g_num_listeners)); + g_num_listeners) > 0); grpc_udp_server_destroy(s, nullptr); @@ -250,7 +250,7 @@ static void test_no_op_with_port_and_socket_factory(void) { addr->sin_family = AF_INET; GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size, snd_buf_size, &handler_factory, - g_num_listeners)); + g_num_listeners) > 0); GPR_ASSERT(socket_factory->number_of_socket_calls == g_num_listeners); GPR_ASSERT(socket_factory->number_of_bind_calls == g_num_listeners); @@ -278,7 +278,7 @@ static void test_no_op_with_port_and_start(void) { addr->sin_family = AF_INET; GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size, snd_buf_size, &handler_factory, - g_num_listeners)); + g_num_listeners) > 0); grpc_udp_server_start(s, nullptr, 0, nullptr); GPR_ASSERT(g_number_of_starts == g_num_listeners); @@ -312,7 +312,7 @@ static void test_receive(int number_of_clients) { addr->ss_family = AF_INET; GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size, snd_buf_size, &handler_factory, - g_num_listeners)); + g_num_listeners) > 0); svrfd = grpc_udp_server_get_fd(s, 0); GPR_ASSERT(svrfd >= 0); From e8b9e95fa275361d1203217a97e01bbadcd211ee Mon Sep 17 00:00:00 2001 From: Hope Casey-Allen Date: Thu, 11 Jul 2019 09:53:13 -0700 Subject: [PATCH 41/55] Use RefCountedPtr instead of raw pointer --- src/core/lib/channel/channelz.h | 2 +- src/core/lib/surface/server.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/channel/channelz.h b/src/core/lib/channel/channelz.h index f8acb531e8a..d26896262ec 100644 --- a/src/core/lib/channel/channelz.h +++ b/src/core/lib/channel/channelz.h @@ -64,7 +64,7 @@ intptr_t GetParentUuidFromArgs(const grpc_channel_args& args); typedef InlinedVector ChildRefsList; class SocketNode; -typedef InlinedVector ChildSocketsList; +typedef InlinedVector, 10> ChildSocketsList; namespace testing { class CallCountingHelperPeer; diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index 8f6df482aa9..6cd86003f01 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -1248,7 +1248,7 @@ void grpc_server_populate_server_sockets( channel_data* c = nullptr; for (c = s->root_channel_data.next; c != &s->root_channel_data; c = c->next) { if (c->socket_node != nullptr && c->socket_node->uuid() >= start_idx) { - server_sockets->push_back(c->socket_node.get()); + server_sockets->push_back(c->socket_node); } } gpr_mu_unlock(&s->mu_global); From 54a3b642466faf6f71578c44615d31af861b92ef Mon Sep 17 00:00:00 2001 From: Arjun Roy Date: Wed, 10 Jul 2019 15:49:37 -0700 Subject: [PATCH 42/55] Pull() eats slice from sbuf wrapped byte buffer. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At present, a commonly used byte buffer wraps an underlying slice buffer. This is used by grpc_call; it passes a byte buffer for send stream ops through the filter stack and down to the transport. Pull() is a one-way operation; given a slice-buffer backed byte-buffer, it starts with a cursor at index=0 and increments it per pull. There is no way for a cursor to be rewinded. Originally Pull() would ref the slice and return a copy; the slice remains in the underlying buffer and is eventually cleaned up (unref'd). Thus, assuming a slice buffer of size N and assuming we pulled all N slices then we have a total of 2N ref/unref pairs. Since Pull() is essentially acting as a consume operation here, and since it is a one-way operation, we simply transfer the ref rather than create a new ref. This cuts down atomic ops. from 2N to N. This reduces streaming latency slightly for streaming fullstack microbenchmarks: BM_StreamingPingPong/0/1 [polls/iter:0 ] 8.87µs ± 1% 8.91µs ± 1% +0.45% (p=0.002 n=20+20) BM_StreamingPingPong/0/2 [polls/iter:0 ] 11.7µs ± 1% 11.7µs ± 1% +0.34% (p=0.002 n=19+19) BM_StreamingPingPong/8/1 [polls/iter:0 ] 9.11µs ± 1% 9.14µs ± 1% +0.29% (p=0.016 n=19+20) BM_StreamingPingPong/64/1 [polls/iter:0 ] 9.29µs ± 1% 9.27µs ± 1% -0.23% (p=0.029 n=18+19) BM_StreamingPingPong/512/2 [polls/iter:0 ] 13.0µs ± 1% 12.9µs ± 1% -0.52% (p=0.000 n=20+19) BM_StreamingPingPong/4096/1 [polls/iter:0 ] 11.8µs ± 1% 11.8µs ± 1% -0.41% (p=0.007 n=20+20) BM_StreamingPingPong/4096/2 [polls/iter:0 ] 17.4µs ± 1% 17.2µs ± 1% -0.73% (p=0.000 n=19+19) BM_StreamingPingPongMsgs/1 [polls/iter:0 ] 2.87µs ± 1% 2.84µs ± 1% -1.18% (p=0.000 n=19+18) BM_StreamingPingPongMsgs/8 [polls/iter:0 ] 2.85µs ± 1% 2.84µs ± 0% -0.38% (p=0.022 n=20+17) BM_StreamingPingPongMsgs/64 [polls/iter:0 ] 3.00µs ± 1% 2.95µs ± 1% -1.64% (p=0.000 n=19+19) BM_StreamingPingPongMsgs/512 [polls/iter:0 ] 3.32µs ± 1% 3.23µs ± 1% -2.62% (p=0.000 n=20+20) BM_StreamingPingPongMsgs/4096 [polls/iter:0 ] 5.38µs ± 2% 5.30µs ± 1% -1.42% (p=0.000 n=19+20) BM_StreamingPingPongMsgs/32768 [polls/iter:0 ] 24.1µs ± 1% 24.0µs ± 0% -0.47% (p=0.000 n=19+18) BM_StreamingPingPong/0/0 [polls/iter:0 ] 5.71µs ± 1% 5.69µs ± 1% -0.36% (p=0.010 n=19+19) BM_StreamingPingPong/0/1 [polls/iter:0 ] 8.64µs ± 1% 8.68µs ± 1% +0.38% (p=0.004 n=20+18) BM_StreamingPingPong/1/2 [polls/iter:0 ] 11.9µs ± 1% 11.8µs ± 1% -0.36% (p=0.003 n=20+18) BM_StreamingPingPong/64/1 [polls/iter:0 ] 9.09µs ± 1% 9.06µs ± 1% -0.32% (p=0.046 n=20+20) BM_StreamingPingPong/64/2 [polls/iter:0 ] 12.2µs ± 0% 12.1µs ± 1% -0.35% (p=0.001 n=19+20) BM_StreamingPingPong/512/2 [polls/iter:0 ] 12.8µs ± 1% 12.7µs ± 1% -0.46% (p=0.000 n=19+19) BM_StreamingPingPong/4096/1 [polls/iter:0 ] 11.7µs ± 2% 11.6µs ± 1% -0.58% (p=0.004 n=20+20) BM_StreamingPingPong/4096/2 [polls/iter:0 ] 17.1µs ± 1% 17.0µs ± 1% -0.55% (p=0.010 n=20+20) BM_StreamingPingPong/32768/1 [polls/iter:0 ] 30.5µs ± 1% 30.6µs ± 0% +0.27% (p=0.003 n=20+19) BM_StreamingPingPong/262144/2 [polls/iter:0 ] 363µs ± 1% 362µs ± 1% -0.22% (p=0.049 n=20+20) BM_StreamingPingPongMsgs/134217728 [polls/iter:4 writes/iter:8.5 ] 266ms ± 0% 263ms ± 1% -1.23% (p=0.029 n=4+4) BM_StreamingPingPongMsgs/64 [polls/iter:0 ] 2.95µs ± 1% 2.93µs ± 1% -0.97% (p=0.000 n=19+20) BM_StreamingPingPongMsgs/512 [polls/iter:0 ] 3.27µs ± 1% 3.19µs ± 1% -2.32% (p=0.000 n=20+20) BM_StreamingPingPongMsgs/4096 [polls/iter:0 ] 5.33µs ± 2% 5.26µs ± 1% -1.30% (p=0.000 n=20+20) BM_StreamingPingPongMsgs/32768 [polls/iter:0 ] 24.1µs ± 1% 24.0µs ± 0% -0.35% (p=0.001 n=19+20) BM_StreamingPingPongWithCoalescingApi/0/2/1 [polls/iter:0 ] 10.6µs ± 1% 10.6µs ± 1% +0.45% (p=0.014 n=20+20) BM_StreamingPingPongWithCoalescingApi/1/2/0 [polls/iter:0 ] 11.3µs ± 1% 11.3µs ± 1% -0.39% (p=0.046 n=20+20) BM_StreamingPingPongWithCoalescingApi/1/1/1 [polls/iter:0 ] 7.92µs ± 1% 7.88µs ± 1% -0.46% (p=0.012 n=19+18) BM_StreamingPingPongWithCoalescingApi/1/2/1 [polls/iter:0 ] 11.0µs ± 1% 10.9µs ± 1% -0.48% (p=0.001 n=18+20) BM_StreamingPingPongWithCoalescingApi/8/1/0 [polls/iter:0 ] 8.33µs ± 1% 8.30µs ± 1% -0.29% (p=0.025 n=19+19) BM_StreamingPingPongWithCoalescingApi/8/2/0 [polls/iter:0 ] 11.3µs ± 1% 11.3µs ± 1% -0.34% (p=0.011 n=19+19) BM_StreamingPingPongWithCoalescingApi/64/1/0 [polls/iter:0 ] 8.49µs ± 1% 8.41µs ± 1% -1.01% (p=0.000 n=20+20) BM_StreamingPingPongWithCoalescingApi/64/2/0 [polls/iter:0 ] 11.6µs ± 1% 11.5µs ± 1% -0.80% (p=0.000 n=19+19) BM_StreamingPingPongWithCoalescingApi/64/1/1 [polls/iter:0 ] 8.06µs ± 1% 8.02µs ± 1% -0.51% (p=0.016 n=20+19) BM_StreamingPingPongWithCoalescingApi/64/2/1 [polls/iter:0 ] 11.2µs ± 1% 11.1µs ± 1% -0.69% (p=0.000 n=19+18) BM_StreamingPingPongWithCoalescingApi/512/1/0 [polls/iter:0 ] 8.83µs ± 1% 8.71µs ± 1% -1.34% (p=0.000 n=20+19) BM_StreamingPingPongWithCoalescingApi/512/2/0 [polls/iter:0 ] 12.2µs ± 1% 12.1µs ± 1% -1.35% (p=0.000 n=19+17) BM_StreamingPingPongWithCoalescingApi/512/1/1 [polls/iter:0 ] 8.40µs ± 1% 8.32µs ± 1% -0.92% (p=0.000 n=20+20) BM_StreamingPingPongWithCoalescingApi/512/2/1 [polls/iter:0 ] 11.9µs ± 1% 11.7µs ± 1% -1.04% (p=0.000 n=19+20) BM_StreamingPingPongWithCoalescingApi/4096/1/0 [polls/iter:0 ] 11.1µs ± 1% 11.0µs ± 1% -0.53% (p=0.002 n=20+19) BM_StreamingPingPongWithCoalescingApi/4096/2/0 [polls/iter:0 ] 16.6µs ± 1% 16.5µs ± 2% -0.47% (p=0.023 n=20+20) BM_StreamingPingPongWithCoalescingApi/32768/1/1 [polls/iter:0 ] 29.6µs ± 1% 29.7µs ± 1% +0.36% (p=0.000 n=20+20) BM_StreamingPingPongWithCoalescingApi/2097152/1/0 [polls/iter:0 ] 1.59ms ± 9% 1.48ms ± 1% -7.02% (p=0.001 n=20+16) BM_StreamingPingPongWithCoalescingApi/2097152/2/0 [polls/iter:0 ] 3.17ms ± 9% 2.95ms ± 0% -6.86% (p=0.015 n=20+16) BM_StreamingPingPongWithCoalescingApi/2097152/1/1 [polls/iter:0 ] 1.59ms ± 9% 1.48ms ± 0% -7.01% (p=0.000 n=20+16) BM_StreamingPingPongWithCoalescingApi/2097152/2/1 [polls/iter:0 ] 3.17ms ± 9% 2.95ms ± 0% -6.91% (p=0.004 n=20+16) BM_StreamingPingPongWithCoalescingApi/134217728/2/1 [polls/iter:0 ] 512ms ± 2% 509ms ± 2% -0.74% (p=0.040 n=20+20) BM_StreamingPingPongWithCoalescingApi/0/1/0 [polls/iter:0 ] 8.07µs ± 1% 8.02µs ± 1% -0.62% (p=0.001 n=20+20) BM_StreamingPingPongWithCoalescingApi/0/2/0 [polls/iter:0 ] 10.9µs ± 1% 10.9µs ± 1% +0.30% (p=0.014 n=20+20) BM_StreamingPingPongWithCoalescingApi/1/1/0 [polls/iter:0 ] 8.27µs ± 1% 8.24µs ± 1% -0.40% (p=0.040 n=20+20) BM_StreamingPingPongWithCoalescingApi/1/1/1 [polls/iter:0 ] 7.86µs ± 1% 7.83µs ± 1% -0.36% (p=0.041 n=20+19) BM_StreamingPingPongWithCoalescingApi/1/2/1 [polls/iter:0 ] 11.0µs ± 1% 10.9µs ± 1% -0.50% (p=0.000 n=18+19) BM_StreamingPingPongWithCoalescingApi/8/2/1 [polls/iter:0 ] 10.9µs ± 1% 10.9µs ± 1% +0.28% (p=0.005 n=17+19) BM_StreamingPingPongWithCoalescingApi/64/1/0 [polls/iter:0 ] 8.43µs ± 1% 8.40µs ± 1% -0.40% (p=0.012 n=20+18) BM_StreamingPingPongWithCoalescingApi/64/2/1 [polls/iter:0 ] 11.2µs ± 1% 11.1µs ± 1% -0.42% (p=0.019 n=20+18) BM_StreamingPingPongWithCoalescingApi/512/1/0 [polls/iter:0 ] 8.77µs ± 1% 8.70µs ± 1% -0.91% (p=0.000 n=20+20) BM_StreamingPingPongWithCoalescingApi/512/2/0 [polls/iter:0 ] 12.2µs ± 1% 12.1µs ± 1% -0.71% (p=0.000 n=20+20) BM_StreamingPingPongWithCoalescingApi/512/2/1 [polls/iter:0 ] 11.8µs ± 1% 11.7µs ± 1% -0.78% (p=0.000 n=20+20) BM_StreamingPingPongWithCoalescingApi/4096/1/0 [polls/iter:0 ] 11.0µs ± 1% 11.0µs ± 1% -0.42% (p=0.030 n=20+20) BM_StreamingPingPongWithCoalescingApi/32768/1/1 [polls/iter:0 ] 29.4µs ± 1% 29.5µs ± 1% +0.35% (p=0.003 n=20+20) BM_StreamingPingPongWithCoalescingApi/2097152/1/0 [polls/iter:0 ] 1.60ms ± 8% 1.48ms ± 1% -7.63% (p=0.002 n=20+16) BM_StreamingPingPongWithCoalescingApi/2097152/2/0 [polls/iter:0 ] 3.20ms ± 8% 2.96ms ± 1% -7.49% (p=0.020 n=20+16) BM_StreamingPingPongWithCoalescingApi/2097152/1/1 [polls/iter:0 ] 1.60ms ± 8% 1.48ms ± 1% -7.68% (p=0.000 n=20+16) BM_StreamingPingPongWithCoalescingApi/2097152/2/1 [polls/iter:0 ] 3.20ms ± 9% 2.95ms ± 1% -7.76% (p=0.001 n=20+16) BM_StreamingPingPongWithCoalescingApi/134217728/1/0 [polls/iter:5.8 writes/iter:9.8 ] 255ms ± 2% 248ms ± 1% -2.59% (p=0.017 n=3+7) BM_StreamingPingPongWithCoalescingApi/134217728/2/1 [polls/iter:9.5 writes/iter:18.5 ] 523ms ± 1% 512ms ± 2% -2.09% (p=0.016 n=5+5) --- src/core/lib/transport/byte_stream.cc | 8 +++----- src/core/lib/transport/byte_stream.h | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/core/lib/transport/byte_stream.cc b/src/core/lib/transport/byte_stream.cc index 16b85ca0db0..1dd234c3761 100644 --- a/src/core/lib/transport/byte_stream.cc +++ b/src/core/lib/transport/byte_stream.cc @@ -55,17 +55,15 @@ void SliceBufferByteStream::Orphan() { bool SliceBufferByteStream::Next(size_t max_size_hint, grpc_closure* on_complete) { - GPR_ASSERT(cursor_ < backing_buffer_.count); + GPR_DEBUG_ASSERT(backing_buffer_.count > 0); return true; } grpc_error* SliceBufferByteStream::Pull(grpc_slice* slice) { - if (shutdown_error_ != GRPC_ERROR_NONE) { + if (GPR_UNLIKELY(shutdown_error_ != GRPC_ERROR_NONE)) { return GRPC_ERROR_REF(shutdown_error_); } - GPR_ASSERT(cursor_ < backing_buffer_.count); - *slice = grpc_slice_ref_internal(backing_buffer_.slices[cursor_]); - ++cursor_; + *slice = grpc_slice_buffer_take_first(&backing_buffer_); return GRPC_ERROR_NONE; } diff --git a/src/core/lib/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h index eff832515da..06503015d2c 100644 --- a/src/core/lib/transport/byte_stream.h +++ b/src/core/lib/transport/byte_stream.h @@ -99,9 +99,8 @@ class SliceBufferByteStream : public ByteStream { void Shutdown(grpc_error* error) override; private: - grpc_slice_buffer backing_buffer_; - size_t cursor_ = 0; grpc_error* shutdown_error_ = GRPC_ERROR_NONE; + grpc_slice_buffer backing_buffer_; }; // From c7793ddddeb2475ea274593dfe0f5436e2046ad6 Mon Sep 17 00:00:00 2001 From: yang-g Date: Thu, 11 Jul 2019 10:16:58 -0700 Subject: [PATCH 43/55] Log transport pointer and goaway last stream id when chttp tracing is enabled --- .../transport/chttp2/transport/chttp2_transport.cc | 12 +++++++++--- .../ext/transport/chttp2/transport/frame_goaway.cc | 2 +- src/core/ext/transport/chttp2/transport/internal.h | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 48c3d002bbd..1b1ec8d3c3a 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1140,6 +1140,7 @@ static void queue_setting_update(grpc_chttp2_transport* t, void grpc_chttp2_add_incoming_goaway(grpc_chttp2_transport* t, uint32_t goaway_error, + uint32_t last_stream_id, const grpc_slice& goaway_text) { // Discard the error from a previous goaway frame (if any) if (t->goaway_error != GRPC_ERROR_NONE) { @@ -1153,6 +1154,9 @@ void grpc_chttp2_add_incoming_goaway(grpc_chttp2_transport* t, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE), GRPC_ERROR_STR_RAW_BYTES, goaway_text); + GRPC_CHTTP2_IF_TRACING( + gpr_log(GPR_INFO, "transport %p got goaway with last stream id %d", t, + last_stream_id)); /* We want to log this irrespective of whether http tracing is enabled */ gpr_log(GPR_INFO, "%s: Got goaway [%d] err=%s", t->peer_string, goaway_error, grpc_error_string(t->goaway_error)); @@ -1191,8 +1195,9 @@ static void maybe_start_some_streams(grpc_chttp2_transport* t) { grpc_chttp2_list_pop_waiting_for_concurrency(t, &s)) { /* safe since we can't (legally) be parsing this stream yet */ GRPC_CHTTP2_IF_TRACING(gpr_log( - GPR_INFO, "HTTP:%s: Allocating new grpc_chttp2_stream %p to id %d", - t->is_client ? "CLI" : "SVR", s, t->next_stream_id)); + GPR_INFO, + "HTTP:%s: Transport %p allocating new grpc_chttp2_stream %p to id %d", + t->is_client ? "CLI" : "SVR", t, s, t->next_stream_id)); GPR_ASSERT(s->id == 0); s->id = t->next_stream_id; @@ -2825,7 +2830,8 @@ static void keepalive_watchdog_fired_locked(void* arg, grpc_error* error) { static void connectivity_state_set(grpc_chttp2_transport* t, grpc_connectivity_state state, const char* reason) { - GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "set connectivity_state=%d", state)); + GRPC_CHTTP2_IF_TRACING( + gpr_log(GPR_INFO, "transport %p set connectivity_state=%d", t, state)); grpc_connectivity_state_set(&t->channel_callback.state_tracker, state, reason); } diff --git a/src/core/ext/transport/chttp2/transport/frame_goaway.cc b/src/core/ext/transport/chttp2/transport/frame_goaway.cc index e901a6bdc76..6dad3497ec6 100644 --- a/src/core/ext/transport/chttp2/transport/frame_goaway.cc +++ b/src/core/ext/transport/chttp2/transport/frame_goaway.cc @@ -139,7 +139,7 @@ grpc_error* grpc_chttp2_goaway_parser_parse(void* parser, p->state = GRPC_CHTTP2_GOAWAY_DEBUG; if (is_last) { grpc_chttp2_add_incoming_goaway( - t, p->error_code, + t, p->error_code, p->last_stream_id, grpc_slice_new(p->debug_data, p->debug_length, gpr_free)); p->debug_data = nullptr; } diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 4ab46f9808b..732bc38a0ac 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -752,6 +752,7 @@ grpc_chttp2_stream* grpc_chttp2_parsing_accept_stream(grpc_chttp2_transport* t, void grpc_chttp2_add_incoming_goaway(grpc_chttp2_transport* t, uint32_t goaway_error, + uint32_t last_stream_id, const grpc_slice& goaway_text); void grpc_chttp2_parsing_become_skip_parser(grpc_chttp2_transport* t); From aa3c2a903d556df2bef204ef579bfa0888751c4b Mon Sep 17 00:00:00 2001 From: Tony Lu Date: Thu, 11 Jul 2019 16:04:44 -0700 Subject: [PATCH 44/55] Reworked hack --- gRPC-Core.podspec | 13 ++++++++++--- templates/gRPC-Core.podspec.template | 16 +++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 340c7b4b66b..f92b78f3fdf 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1387,8 +1387,15 @@ Pod::Spec.new do |s| # TODO (mxyan): Instead of this hack, add include path "third_party" to C core's include path? s.prepare_command = <<-END_OF_COMMAND - find src/core/ -type f ! -path '*.grpc_back' -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include "(pb(_.*)?\\.h)";#include ;g' - find src/core/ -type f -path '*.grpc_back' -print0 | xargs -0 rm - find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include \\ +#else\\ + #include "\\1"\\ +#endif;g' + find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i '' 's;#include ;#if COCOAPODS\\ + #include \\ +#else\\ + #include \\ +#endif;g' END_OF_COMMAND end diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 89a1e9188ff..a22ce9c33e9 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -211,9 +211,15 @@ end # TODO (mxyan): Instead of this hack, add include path "third_party" to C core's include path? - s.prepare_command = <<-END_OF_COMMAND - find src/core/ -type f ! -path '*.grpc_back' -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include "(pb(_.*)?\\.h)";#include ;g' - find src/core/ -type f -path '*.grpc_back' -print0 | xargs -0 rm - find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i'.grpc_back' 's;#include \\ +#else\\ + #include "\\1"\\ +#endif;g' + find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i '' 's;#include ;#if COCOAPODS\\ + #include \\ +#else\\ + #include \\ +#endif;g' end From 4999420c7d980189ecc11d5e58a9387ef7573400 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 11 Jul 2019 16:11:15 -0700 Subject: [PATCH 45/55] Add a test client for certain grpclb fallback scenarios --- CMakeLists.txt | 63 +++++ Makefile | 58 ++++ build.yaml | 15 ++ src/proto/grpc/testing/messages.proto | 20 ++ test/cpp/interop/BUILD | 16 ++ test/cpp/interop/grpclb_fallback_test.cc | 254 ++++++++++++++++++ .../generated/sources_and_headers.json | 29 ++ tools/run_tests/generated/tests.json | 24 ++ 8 files changed, 479 insertions(+) create mode 100644 test/cpp/interop/grpclb_fallback_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 8dfa4e8009a..fc67b4512ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -638,6 +638,7 @@ add_dependencies(buildtests_cxx grpc_linux_system_roots_test) add_dependencies(buildtests_cxx grpc_tool_test) add_dependencies(buildtests_cxx grpclb_api_test) add_dependencies(buildtests_cxx grpclb_end2end_test) +add_dependencies(buildtests_cxx grpclb_fallback_test) add_dependencies(buildtests_cxx h2_ssl_cert_test) add_dependencies(buildtests_cxx h2_ssl_session_reuse_test) add_dependencies(buildtests_cxx health_service_end2end_test) @@ -14465,6 +14466,68 @@ target_link_libraries(grpclb_end2end_test ) +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + +add_executable(grpclb_fallback_test + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/empty.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/empty.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/empty.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/empty.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/messages.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/messages.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/messages.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/messages.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/test.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/test.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/test.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/test.grpc.pb.h + test/cpp/interop/grpclb_fallback_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + +protobuf_generate_grpc_cpp( + src/proto/grpc/testing/empty.proto +) +protobuf_generate_grpc_cpp( + src/proto/grpc/testing/messages.proto +) +protobuf_generate_grpc_cpp( + src/proto/grpc/testing/test.proto +) + +target_include_directories(grpclb_fallback_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(grpclb_fallback_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr + grpc++_test_config + ${_gRPC_GFLAGS_LIBRARIES} +) + + endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/Makefile b/Makefile index 2a1bfad1f35..952b20155fd 100644 --- a/Makefile +++ b/Makefile @@ -1228,6 +1228,7 @@ grpc_ruby_plugin: $(BINDIR)/$(CONFIG)/grpc_ruby_plugin grpc_tool_test: $(BINDIR)/$(CONFIG)/grpc_tool_test grpclb_api_test: $(BINDIR)/$(CONFIG)/grpclb_api_test grpclb_end2end_test: $(BINDIR)/$(CONFIG)/grpclb_end2end_test +grpclb_fallback_test: $(BINDIR)/$(CONFIG)/grpclb_fallback_test h2_ssl_cert_test: $(BINDIR)/$(CONFIG)/h2_ssl_cert_test h2_ssl_session_reuse_test: $(BINDIR)/$(CONFIG)/h2_ssl_session_reuse_test health_service_end2end_test: $(BINDIR)/$(CONFIG)/health_service_end2end_test @@ -1697,6 +1698,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/grpclb_end2end_test \ + $(BINDIR)/$(CONFIG)/grpclb_fallback_test \ $(BINDIR)/$(CONFIG)/h2_ssl_cert_test \ $(BINDIR)/$(CONFIG)/h2_ssl_session_reuse_test \ $(BINDIR)/$(CONFIG)/health_service_end2end_test \ @@ -1862,6 +1864,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/grpclb_end2end_test \ + $(BINDIR)/$(CONFIG)/grpclb_fallback_test \ $(BINDIR)/$(CONFIG)/h2_ssl_cert_test \ $(BINDIR)/$(CONFIG)/h2_ssl_session_reuse_test \ $(BINDIR)/$(CONFIG)/health_service_end2end_test \ @@ -2366,6 +2369,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/grpclb_api_test || ( echo test grpclb_api_test failed ; exit 1 ) $(E) "[RUN] Testing grpclb_end2end_test" $(Q) $(BINDIR)/$(CONFIG)/grpclb_end2end_test || ( echo test grpclb_end2end_test failed ; exit 1 ) + $(E) "[RUN] Testing grpclb_fallback_test" + $(Q) $(BINDIR)/$(CONFIG)/grpclb_fallback_test || ( echo test grpclb_fallback_test failed ; exit 1 ) $(E) "[RUN] Testing h2_ssl_cert_test" $(Q) $(BINDIR)/$(CONFIG)/h2_ssl_cert_test || ( echo test h2_ssl_cert_test failed ; exit 1 ) $(E) "[RUN] Testing h2_ssl_session_reuse_test" @@ -17488,6 +17493,59 @@ endif $(OBJDIR)/$(CONFIG)/test/cpp/end2end/grpclb_end2end_test.o: $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc +GRPCLB_FALLBACK_TEST_SRC = \ + $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc \ + $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc \ + $(GENDIR)/src/proto/grpc/testing/test.pb.cc $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc \ + test/cpp/interop/grpclb_fallback_test.cc \ + +GRPCLB_FALLBACK_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPCLB_FALLBACK_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpclb_fallback_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+. + +$(BINDIR)/$(CONFIG)/grpclb_fallback_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/grpclb_fallback_test: $(PROTOBUF_DEP) $(GRPCLB_FALLBACK_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(GRPCLB_FALLBACK_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpclb_fallback_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/empty.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/messages.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +$(OBJDIR)/$(CONFIG)/test/cpp/interop/grpclb_fallback_test.o: $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a + +deps_grpclb_fallback_test: $(GRPCLB_FALLBACK_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPCLB_FALLBACK_TEST_OBJS:.o=.dep) +endif +endif +$(OBJDIR)/$(CONFIG)/test/cpp/interop/grpclb_fallback_test.o: $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/test.pb.cc $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc + + H2_SSL_CERT_TEST_SRC = \ test/core/end2end/h2_ssl_cert_test.cc \ diff --git a/build.yaml b/build.yaml index 5915b9ccd64..f42efbb3b21 100644 --- a/build.yaml +++ b/build.yaml @@ -5041,6 +5041,21 @@ targets: - grpc++ - grpc - gpr +- name: grpclb_fallback_test + build: test + language: c++ + src: + - src/proto/grpc/testing/empty.proto + - src/proto/grpc/testing/messages.proto + - src/proto/grpc/testing/test.proto + - test/cpp/interop/grpclb_fallback_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr + - grpc++_test_config - name: h2_ssl_cert_test gtest: true build: test diff --git a/src/proto/grpc/testing/messages.proto b/src/proto/grpc/testing/messages.proto index 3b966ae1b25..4fed83fb389 100644 --- a/src/proto/grpc/testing/messages.proto +++ b/src/proto/grpc/testing/messages.proto @@ -48,6 +48,21 @@ message EchoStatus { string message = 2; } +// The type of route that a client took to reach a server w.r.t. gRPCLB. +// The server must fill in "fallback" if it detects that the RPC reached +// the server via the "gRPCLB fallback" path, and "backend" if it detects +// that the RPC reached the server via "gRPCLB backend" path (i.e. if it got +// the address of this server from the gRPCLB server BalanceLoad RPC). Exactly +// how this detection is done is context and server dependant. +enum GrpclbRouteType { + // Server didn't detect the route that a client took to reach it. + GRPCLB_ROUTE_TYPE_UNKNOWN = 0; + // Indicates that a client reached a server via gRPCLB fallback. + GRPCLB_ROUTE_TYPE_FALLBACK = 1; + // Indicates that a client reached a server as a gRPCLB-given backend. + GRPCLB_ROUTE_TYPE_BACKEND = 2; +} + // Unary request. message SimpleRequest { // Desired payload type in the response from the server. @@ -80,6 +95,9 @@ message SimpleRequest { // Whether SimpleResponse should include server_id. bool fill_server_id = 9; + + // Whether SimpleResponse should include grpclb_route_type. + bool fill_grpclb_route_type = 10; } // Unary response, as configured by the request. @@ -95,6 +113,8 @@ message SimpleResponse { // Server ID. This must be unique among different server instances, // but the same across all RPC's made to a particular server instance. string server_id = 4; + // gRPCLB Path. + GrpclbRouteType grpclb_route_type = 5; } // Client-streaming request. diff --git a/test/cpp/interop/BUILD b/test/cpp/interop/BUILD index 802302bc8bf..e575ef88c5d 100644 --- a/test/cpp/interop/BUILD +++ b/test/cpp/interop/BUILD @@ -38,6 +38,22 @@ grpc_cc_library( ], ) +grpc_cc_binary( + name = "grpclb_fallback_test", + srcs = [ + "grpclb_fallback_test.cc", + ], + language = "C++", + deps = [ + "//src/proto/grpc/testing:empty_proto", + "//src/proto/grpc/testing:messages_proto", + "//src/proto/grpc/testing:test_proto", + "//test/cpp/util:test_config", + "//test/cpp/util:test_util", + "//:grpc++", + ], +) + grpc_cc_binary( name = "interop_server", srcs = [ diff --git a/test/cpp/interop/grpclb_fallback_test.cc b/test/cpp/interop/grpclb_fallback_test.cc new file mode 100644 index 00000000000..ddfad7e01b1 --- /dev/null +++ b/test/cpp/interop/grpclb_fallback_test.cc @@ -0,0 +1,254 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/iomgr/socket_mutator.h" +#include "src/proto/grpc/testing/empty.pb.h" +#include "src/proto/grpc/testing/messages.pb.h" +#include "src/proto/grpc/testing/test.grpc.pb.h" +#include "src/proto/grpc/testing/test.pb.h" + +#include "test/cpp/util/test_config.h" +#include "test/cpp/util/test_credentials_provider.h" + +DEFINE_string(custom_credentials_type, "", "User provided credentials type."); +DEFINE_string(server_uri, "localhost:1000", "Server URI target"); +DEFINE_string(unroute_lb_and_backend_addrs_cmd, "exit 1", + "Shell command used to make LB and backend addresses unroutable"); +DEFINE_string(blackhole_lb_and_backend_addrs_cmd, "exit 1", + "Shell command used to make LB and backend addresses blackholed"); +DEFINE_string( + test_case, "", + "Test case to run. Valid options are:\n\n" + "fast_fallback_before_startup : fallback before establishing connection to " + "LB;\n" + "fast_fallback_after_startup : fallback after startup due to LB/backend " + "addresses becoming unroutable;\n" + "slow_fallback_before_startup : fallback before startup due to LB address " + "being blackholed;\n" + "slow_fallback_after_startup : fallback after startup due to LB/backend " + "addresses becoming blackholed;\n"); + +using grpc::testing::GrpclbRouteType; +using grpc::testing::SimpleRequest; +using grpc::testing::SimpleResponse; +using grpc::testing::TestService; + +namespace { + +enum RpcMode { + FailFast, + WaitForReady, +}; + +GrpclbRouteType DoRPCAndGetPath(TestService::Stub* stub, int deadline_seconds, + RpcMode rpc_mode) { + gpr_log(GPR_INFO, "DoRPCAndGetPath deadline_seconds:%d rpc_mode:%d", + deadline_seconds, rpc_mode); + SimpleRequest request; + SimpleResponse response; + grpc::ClientContext context; + if (rpc_mode == WaitForReady) { + context.set_wait_for_ready(true); + } + request.set_fill_grpclb_route_type(true); + std::chrono::system_clock::time_point deadline = + std::chrono::system_clock::now() + std::chrono::seconds(deadline_seconds); + context.set_deadline(deadline); + grpc::Status s = stub->UnaryCall(&context, request, &response); + if (!s.ok()) { + gpr_log(GPR_INFO, "DoRPCAndGetPath failed. status-message: %s", + s.error_message().c_str()); + return GrpclbRouteType::GRPCLB_ROUTE_TYPE_UNKNOWN; + } + GPR_ASSERT(response.grpclb_route_type() == + GrpclbRouteType::GRPCLB_ROUTE_TYPE_BACKEND || + response.grpclb_route_type() == + GrpclbRouteType::GRPCLB_ROUTE_TYPE_FALLBACK); + gpr_log(GPR_INFO, "DoRPCAndGetPath done. grpclb_route_type:%d", + response.grpclb_route_type()); + return response.grpclb_route_type(); +} + +GrpclbRouteType DoRPCAndGetPath(TestService::Stub* stub, int deadline_seconds) { + return DoRPCAndGetPath(stub, deadline_seconds, FailFast); +} + +GrpclbRouteType DoWaitForReadyRPCAndGetPath(TestService::Stub* stub, + int deadline_seconds) { + return DoRPCAndGetPath(stub, deadline_seconds, WaitForReady); +} + +bool TcpUserTimeoutMutateFd(int fd, grpc_socket_mutator* mutator) { + int timeout = 20000; // 20 seconds + gpr_log(GPR_INFO, "Setting socket option TCP_USER_TIMEOUT on fd: %d", fd); + if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &timeout, + sizeof(timeout))) { + gpr_log(GPR_ERROR, "Failed to set socket option TCP_USER_TIMEOUT"); + abort(); + } + int newval; + socklen_t len = sizeof(newval); + if (0 != getsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &newval, &len) || + newval != timeout) { + gpr_log(GPR_ERROR, "Failed to set socket option TCP_USER_TIMEOUT"); + abort(); + } + return true; +} + +int TcpUserTimeoutCompare(grpc_socket_mutator* a, grpc_socket_mutator* b) { + return 0; +} + +void TcpUserTimeoutDestroy(grpc_socket_mutator* mutator) { gpr_free(mutator); } + +const grpc_socket_mutator_vtable kTcpUserTimeoutMutatorVtable = + grpc_socket_mutator_vtable{ + .mutate_fd = TcpUserTimeoutMutateFd, + .compare = TcpUserTimeoutCompare, + .destroy = TcpUserTimeoutDestroy, + }; + +std::unique_ptr CreateFallbackTestStub() { + grpc::ChannelArguments channel_args; + grpc_socket_mutator* tcp_user_timeout_mutator = + static_cast( + gpr_malloc(sizeof(tcp_user_timeout_mutator))); + grpc_socket_mutator_init(tcp_user_timeout_mutator, + &kTcpUserTimeoutMutatorVtable); + channel_args.SetSocketMutator(tcp_user_timeout_mutator); + // Allow LB policy to be configured by service config + channel_args.SetInt(GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION, 0); + std::shared_ptr channel_creds = + grpc::testing::GetCredentialsProvider()->GetChannelCredentials( + FLAGS_custom_credentials_type, &channel_args); + return TestService::NewStub( + grpc::CreateCustomChannel(FLAGS_server_uri, channel_creds, channel_args)); +} + +void RunCommand(const std::string command) { + gpr_log(GPR_INFO, "RunCommand: |%s|", command.c_str()); + int out = std::system(command.c_str()); + if (WIFEXITED(out)) { + int code = WEXITSTATUS(out); + if (code != 0) { + gpr_log(GPR_ERROR, "RunCommand failed exit code:%d command:|%s|", code, + command.c_str()); + abort(); + } + } else { + gpr_log(GPR_ERROR, "RunCommand failed command:|%s|", command.c_str()); + abort(); + } +} + +void RunFallbackBeforeStartupTest( + const std::string break_lb_and_backend_conns_cmd, + int per_rpc_deadline_seconds) { + std::unique_ptr stub = CreateFallbackTestStub(); + RunCommand(break_lb_and_backend_conns_cmd); + for (size_t i = 0; i < 30; i++) { + GrpclbRouteType grpclb_route_type = + DoRPCAndGetPath(stub.get(), per_rpc_deadline_seconds); + if (grpclb_route_type != GrpclbRouteType::GRPCLB_ROUTE_TYPE_FALLBACK) { + gpr_log(GPR_ERROR, "Expected grpclb route type: FALLBACK. Got: %d", + grpclb_route_type); + abort(); + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } +} + +void DoFastFallbackBeforeStartup() { + RunFallbackBeforeStartupTest(FLAGS_unroute_lb_and_backend_addrs_cmd, 9); +} + +void DoSlowFallbackBeforeStartup() { + RunFallbackBeforeStartupTest(FLAGS_blackhole_lb_and_backend_addrs_cmd, 20); +} + +void RunFallbackAfterStartupTest( + const std::string break_lb_and_backend_conns_cmd) { + std::unique_ptr stub = CreateFallbackTestStub(); + GrpclbRouteType grpclb_route_type = DoRPCAndGetPath(stub.get(), 20); + if (grpclb_route_type != GrpclbRouteType::GRPCLB_ROUTE_TYPE_BACKEND) { + gpr_log(GPR_ERROR, "Expected grpclb route type: BACKEND. Got: %d", + grpclb_route_type); + abort(); + } + RunCommand(break_lb_and_backend_conns_cmd); + for (size_t i = 0; i < 40; i++) { + GrpclbRouteType grpclb_route_type = + DoWaitForReadyRPCAndGetPath(stub.get(), 1); + // Backends should be unreachable by now, otherwise the test is broken. + GPR_ASSERT(grpclb_route_type != GrpclbRouteType::GRPCLB_ROUTE_TYPE_BACKEND); + if (grpclb_route_type == GrpclbRouteType::GRPCLB_ROUTE_TYPE_FALLBACK) { + gpr_log(GPR_INFO, + "Made one successul RPC to a fallback. Now expect the same for " + "the rest."); + break; + } else { + gpr_log(GPR_ERROR, "Retryable RPC failure on iteration: %" PRIdPTR, i); + } + } + for (size_t i = 0; i < 30; i++) { + GrpclbRouteType grpclb_route_type = DoRPCAndGetPath(stub.get(), 20); + if (grpclb_route_type != GrpclbRouteType::GRPCLB_ROUTE_TYPE_FALLBACK) { + gpr_log(GPR_ERROR, "Expected grpclb route type: FALLBACK. Got: %d", + grpclb_route_type); + abort(); + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } +} + +void DoFastFallbackAfterStartup() { + RunFallbackAfterStartupTest(FLAGS_unroute_lb_and_backend_addrs_cmd); +} + +void DoSlowFallbackAfterStartup() { + RunFallbackAfterStartupTest(FLAGS_blackhole_lb_and_backend_addrs_cmd); +} +} // namespace + +int main(int argc, char** argv) { + grpc::testing::InitTest(&argc, &argv, true); + gpr_log(GPR_INFO, "Testing: %s", FLAGS_test_case.c_str()); + if (FLAGS_test_case == "fast_fallback_before_startup") { + DoFastFallbackBeforeStartup(); + gpr_log(GPR_INFO, "DoFastFallbackBeforeStartup done!"); + } else if (FLAGS_test_case == "slow_fallback_before_startup") { + DoSlowFallbackBeforeStartup(); + gpr_log(GPR_INFO, "DoSlowFallbackBeforeStartup done!"); + } else if (FLAGS_test_case == "fast_fallback_after_startup") { + DoFastFallbackAfterStartup(); + gpr_log(GPR_INFO, "DoFastFallbackAfterStartup done!"); + } else if (FLAGS_test_case == "slow_fallback_after_startup") { + DoSlowFallbackAfterStartup(); + gpr_log(GPR_INFO, "DoSlowFallbackAfterStartup done!"); + } else { + gpr_log(GPR_ERROR, "Invalid test case: %s", FLAGS_test_case.c_str()); + abort(); + } +} diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 7a1baae36cd..0b56c7b32df 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -4037,6 +4037,35 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "grpc", + "grpc++", + "grpc++_test_config", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [ + "src/proto/grpc/testing/empty.grpc.pb.h", + "src/proto/grpc/testing/empty.pb.h", + "src/proto/grpc/testing/empty_mock.grpc.pb.h", + "src/proto/grpc/testing/messages.grpc.pb.h", + "src/proto/grpc/testing/messages.pb.h", + "src/proto/grpc/testing/messages_mock.grpc.pb.h", + "src/proto/grpc/testing/test.grpc.pb.h", + "src/proto/grpc/testing/test.pb.h", + "src/proto/grpc/testing/test_mock.grpc.pb.h" + ], + "is_filegroup": false, + "language": "c++", + "name": "grpclb_fallback_test", + "src": [ + "test/cpp/interop/grpclb_fallback_test.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 2ac529238d5..fe10187fffb 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4813,6 +4813,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "grpclb_fallback_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, From 34a688050857ee29e413ec3baf833873d9c36bdf Mon Sep 17 00:00:00 2001 From: Tony Lu Date: Thu, 11 Jul 2019 18:31:51 -0700 Subject: [PATCH 46/55] Fixing formatting issues --- gRPC-Core.podspec | 12 ++---------- templates/gRPC-Core.podspec.template | 15 ++++----------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index f92b78f3fdf..2dfcb311b5d 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -1387,15 +1387,7 @@ Pod::Spec.new do |s| # TODO (mxyan): Instead of this hack, add include path "third_party" to C core's include path? s.prepare_command = <<-END_OF_COMMAND - find src/core/ -type f -print0 | xargs -0 -L1 sed -E -i '' 's;#include "(pb(_.*)?\\.h)";#if COCOAPODS\\ - #include \\ -#else\\ - #include "\\1"\\ -#endif;g' - find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i '' 's;#include ;#if COCOAPODS\\ - #include \\ -#else\\ - #include \\ -#endif;g' + find src/core/ -type f -print0 | xargs -0 -L1 sed -E -i '' 's;#include "(pb(_.*)?\\.h)";#if COCOAPODS\\\n #include \\\n#else\\\n #include "\\1"\\\n#endif;g' + find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i '' 's;#include ;#if COCOAPODS\\\n #include \\\n#else\\\n #include \\\n#endif;g' END_OF_COMMAND end diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index a22ce9c33e9..652c4ed43f0 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -211,15 +211,8 @@ end # TODO (mxyan): Instead of this hack, add include path "third_party" to C core's include path? - s.prepare_command = <<-END_OF_COMMAND - find src/core/ -type f -print0 | xargs -0 -L1 sed -E -i '' 's;#include "(pb(_.*)?\\.h)";#if COCOAPODS\\ - #include \\ -#else\\ - #include "\\1"\\ -#endif;g' - find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i '' 's;#include ;#if COCOAPODS\\ - #include \\ -#else\\ - #include \\ -#endif;g' + s.prepare_command = <<-END_OF_COMMAND + find src/core/ -type f -print0 | xargs -0 -L1 sed -E -i '' 's;#include "(pb(_.*)?\\.h)";#if COCOAPODS\\\n #include \\\n#else\\\n #include "\\1"\\\n#endif;g' + find src/core/ -type f \\( -path '*.h' -or -path '*.cc' \\) -print0 | xargs -0 -L1 sed -E -i '' 's;#include ;#if COCOAPODS\\\n #include \\\n#else\\\n #include \\\n#endif;g' + END_OF_COMMAND end From 3958a53bf7ba8a847154f55bf1eb50ca503fb220 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 11 Jul 2019 19:08:27 -0700 Subject: [PATCH 47/55] Address review comments; fix sanity --- CMakeLists.txt | 4 ++++ build.yaml | 2 ++ test/cpp/interop/grpclb_fallback_test.cc | 26 ++++++++++++++++++++---- tools/run_tests/generated/tests.json | 10 ++------- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc67b4512ea..a68f5caff81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -638,7 +638,9 @@ add_dependencies(buildtests_cxx grpc_linux_system_roots_test) add_dependencies(buildtests_cxx grpc_tool_test) add_dependencies(buildtests_cxx grpclb_api_test) add_dependencies(buildtests_cxx grpclb_end2end_test) +if(_gRPC_PLATFORM_LINUX) add_dependencies(buildtests_cxx grpclb_fallback_test) +endif() add_dependencies(buildtests_cxx h2_ssl_cert_test) add_dependencies(buildtests_cxx h2_ssl_session_reuse_test) add_dependencies(buildtests_cxx health_service_end2end_test) @@ -14468,6 +14470,7 @@ target_link_libraries(grpclb_end2end_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX) add_executable(grpclb_fallback_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/empty.pb.cc @@ -14528,6 +14531,7 @@ target_link_libraries(grpclb_fallback_test ) +endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/build.yaml b/build.yaml index f42efbb3b21..c36b3c72594 100644 --- a/build.yaml +++ b/build.yaml @@ -5056,6 +5056,8 @@ targets: - grpc - gpr - grpc++_test_config + platforms: + - linux - name: h2_ssl_cert_test gtest: true build: test diff --git a/test/cpp/interop/grpclb_fallback_test.cc b/test/cpp/interop/grpclb_fallback_test.cc index ddfad7e01b1..3622482b67d 100644 --- a/test/cpp/interop/grpclb_fallback_test.cc +++ b/test/cpp/interop/grpclb_fallback_test.cc @@ -1,3 +1,21 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + #include #include @@ -111,7 +129,7 @@ bool TcpUserTimeoutMutateFd(int fd, grpc_socket_mutator* mutator) { socklen_t len = sizeof(newval); if (0 != getsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &newval, &len) || newval != timeout) { - gpr_log(GPR_ERROR, "Failed to set socket option TCP_USER_TIMEOUT"); + gpr_log(GPR_ERROR, "Failed to get expected socket option TCP_USER_TIMEOUT"); abort(); } return true; @@ -147,7 +165,7 @@ std::unique_ptr CreateFallbackTestStub() { grpc::CreateCustomChannel(FLAGS_server_uri, channel_creds, channel_args)); } -void RunCommand(const std::string command) { +void RunCommand(const std::string& command) { gpr_log(GPR_INFO, "RunCommand: |%s|", command.c_str()); int out = std::system(command.c_str()); if (WIFEXITED(out)) { @@ -164,7 +182,7 @@ void RunCommand(const std::string command) { } void RunFallbackBeforeStartupTest( - const std::string break_lb_and_backend_conns_cmd, + const std::string& break_lb_and_backend_conns_cmd, int per_rpc_deadline_seconds) { std::unique_ptr stub = CreateFallbackTestStub(); RunCommand(break_lb_and_backend_conns_cmd); @@ -189,7 +207,7 @@ void DoSlowFallbackBeforeStartup() { } void RunFallbackAfterStartupTest( - const std::string break_lb_and_backend_conns_cmd) { + const std::string& break_lb_and_backend_conns_cmd) { std::unique_ptr stub = CreateFallbackTestStub(); GrpclbRouteType grpclb_route_type = DoRPCAndGetPath(stub.get(), 20); if (grpclb_route_type != GrpclbRouteType::GRPCLB_ROUTE_TYPE_BACKEND) { diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index fe10187fffb..f4757e5ef62 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4817,10 +4817,7 @@ "args": [], "benchmark": false, "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" + "linux" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -4830,10 +4827,7 @@ "language": "c++", "name": "grpclb_fallback_test", "platforms": [ - "linux", - "mac", - "posix", - "windows" + "linux" ], "uses_polling": true }, From a0d9ec81a5503c5d71f317b66682748c5eb34852 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 12 Jul 2019 14:11:26 -0700 Subject: [PATCH 48/55] Add a sanity check for the Python release process. This fixes https://github.com/grpc/grpc/issues/19632. --- tools/release/verify_python_release.py | 116 +++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 tools/release/verify_python_release.py diff --git a/tools/release/verify_python_release.py b/tools/release/verify_python_release.py new file mode 100644 index 00000000000..af2892b61a5 --- /dev/null +++ b/tools/release/verify_python_release.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +#Copyright 2019 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Verifies that all gRPC Python artifacts have been successfully published. + +This script is intended to be run from a directory containing the artifacts +that have been uploaded and only the artifacts that have been uploaded. We use +PyPI's JSON API to verify that the proper filenames and checksums are present. + +Note that PyPI may take several minutes to update its metadata. Don't have a +heart attack immediately. + +This sanity check is a good first step, but ideally, we would automate the +entire release process. +""" + +import argparse +import collections +import hashlib +import os +import requests +import sys + +_DEFAULT_PACKAGES = [ + "grpcio", + "grpcio-tools", + "grpcio-status", + "grpcio-health-checking", + "grpcio-reflection", + "grpcio-channelz", + "grpcio-testing", +] + +Artifact = collections.namedtuple("Artifact", ("filename", "checksum")) + + +def _get_md5_checksum(filename): + """Calculate the md5sum for a file.""" + hash_md5 = hashlib.md5() + with open(filename, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +def _get_local_artifacts(): + """Get a set of artifacts representing all files in the cwd.""" + return set( + Artifact(f, _get_md5_checksum(f)) for f in os.listdir(os.getcwd())) + + +def _get_remote_artifacts_for_package(package, version): + """Get a list of artifacts based on PyPi's json metadata. + + Note that this data will not updated immediately after upload. In my + experience, it has taken a minute on average to be fresh. + """ + artifacts = set() + payload = requests.get("https://pypi.org/pypi/{}/{}/json".format( + package, version)).json() + for download_info in payload['releases'][version]: + artifacts.add( + Artifact(download_info['filename'], download_info['md5_digest'])) + return artifacts + + +def _get_remote_artifacts_for_packages(packages, version): + artifacts = set() + for package in packages: + artifacts |= _get_remote_artifacts_for_package(package, version) + return artifacts + + +def _verify_release(version, packages): + """Compare the local artifacts to the packages uploaded to PyPI.""" + local_artifacts = _get_local_artifacts() + remote_artifacts = _get_remote_artifacts_for_packages(packages, version) + if local_artifacts != remote_artifacts: + local_but_not_remote = local_artifacts - remote_artifacts + remote_but_not_local = remote_artifacts - local_artifacts + if local_but_not_remote: + print("The following artifacts exist locally but not remotely.") + for artifact in local_but_not_remote: + print(artifact) + if remote_but_not_local: + print("The following artifacts exist remotely but not locally.") + for artifact in remote_but_not_local: + print(artifact) + sys.exit(1) + print("Release verified successfully.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + "Verify a release. Run this from a directory containing only the" + "artifacts to be uploaded. Note that PyPI may take several minutes" + "after the upload to reflect the proper metadata." + ) + parser.add_argument("version") + parser.add_argument( + "packages", nargs='*', type=str, default=_DEFAULT_PACKAGES) + args = parser.parse_args() + _verify_release(args.version, args.packages) From 994985be97e59e6379655988a1af56b977749d58 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 12 Jul 2019 10:27:47 -0700 Subject: [PATCH 49/55] Log refcount traces if tracer enabled, even if DEBUG_LOCATION is not used. --- src/core/lib/gprpp/debug_location.h | 5 +- src/core/lib/gprpp/ref_counted.h | 84 +++++++++++++++++++++-------- 2 files changed, 66 insertions(+), 23 deletions(-) diff --git a/src/core/lib/gprpp/debug_location.h b/src/core/lib/gprpp/debug_location.h index 287761beafb..d2384fde289 100644 --- a/src/core/lib/gprpp/debug_location.h +++ b/src/core/lib/gprpp/debug_location.h @@ -25,10 +25,12 @@ namespace grpc_core { // No-op for non-debug builds. // Callers can use the DEBUG_LOCATION macro in either case. #ifndef NDEBUG +// TODO(roth): See if there's a way to automatically populate this, +// similarly to how absl::SourceLocation::current() works, so that +// callers don't need to explicitly pass DEBUG_LOCATION anywhere. class DebugLocation { public: DebugLocation(const char* file, int line) : file_(file), line_(line) {} - bool Log() const { return true; } const char* file() const { return file_; } int line() const { return line_; } @@ -40,7 +42,6 @@ class DebugLocation { #else class DebugLocation { public: - bool Log() const { return false; } const char* file() const { return nullptr; } int line() const { return -1; } }; diff --git a/src/core/lib/gprpp/ref_counted.h b/src/core/lib/gprpp/ref_counted.h index cab1aaaaab1..5c7a5cf0735 100644 --- a/src/core/lib/gprpp/ref_counted.h +++ b/src/core/lib/gprpp/ref_counted.h @@ -89,72 +89,114 @@ class RefCount { } // Increases the ref-count by `n`. - void Ref(Value n = 1) { value_.FetchAdd(n, MemoryOrder::RELAXED); } + void Ref(Value n = 1) { +#ifndef NDEBUG + const Value prior = value_.FetchAdd(n, MemoryOrder::RELAXED); + if (trace_flag_ != nullptr && trace_flag_->enabled()) { + gpr_log(GPR_INFO, "%s:%p ref %" PRIdPTR " -> %" PRIdPTR, + trace_flag_->name(), this, prior, prior + n); + } +#else + value_.FetchAdd(n, MemoryOrder::RELAXED); +#endif + } void Ref(const DebugLocation& location, const char* reason, Value n = 1) { #ifndef NDEBUG - if (location.Log() && trace_flag_ != nullptr && trace_flag_->enabled()) { - const RefCount::Value old_refs = get(); + const Value prior = value_.FetchAdd(n, MemoryOrder::RELAXED); + if (trace_flag_ != nullptr && trace_flag_->enabled()) { gpr_log(GPR_INFO, "%s:%p %s:%d ref %" PRIdPTR " -> %" PRIdPTR " %s", trace_flag_->name(), this, location.file(), location.line(), - old_refs, old_refs + n, reason); + prior, prior + n, reason); } +#else + value_.FetchAdd(n, MemoryOrder::RELAXED); #endif - Ref(n); } // Similar to Ref() with an assert on the ref-count being non-zero. void RefNonZero() { #ifndef NDEBUG const Value prior = value_.FetchAdd(1, MemoryOrder::RELAXED); + if (trace_flag_ != nullptr && trace_flag_->enabled()) { + gpr_log(GPR_INFO, "%s:%p ref %" PRIdPTR " -> %" PRIdPTR, + trace_flag_->name(), this, prior, prior + 1); + } assert(prior > 0); #else - Ref(); + value_.FetchAdd(1, MemoryOrder::RELAXED); #endif } void RefNonZero(const DebugLocation& location, const char* reason) { #ifndef NDEBUG - if (location.Log() && trace_flag_ != nullptr && trace_flag_->enabled()) { - const RefCount::Value old_refs = get(); + const Value prior = value_.FetchAdd(1, MemoryOrder::RELAXED); + if (trace_flag_ != nullptr && trace_flag_->enabled()) { gpr_log(GPR_INFO, "%s:%p %s:%d ref %" PRIdPTR " -> %" PRIdPTR " %s", trace_flag_->name(), this, location.file(), location.line(), - old_refs, old_refs + 1, reason); + prior, prior + 1, reason); } -#endif + assert(prior > 0); +#else RefNonZero(); +#endif } - bool RefIfNonZero() { return value_.IncrementIfNonzero(); } - + bool RefIfNonZero() { +#ifndef NDEBUG + if (trace_flag_ != nullptr && trace_flag_->enabled()) { + const Value prior = get(); + gpr_log(GPR_INFO, "%s:%p ref_if_non_zero %" PRIdPTR " -> %" PRIdPTR, + trace_flag_->name(), this, prior, prior + 1); + } +#endif + return value_.IncrementIfNonzero(); + } bool RefIfNonZero(const DebugLocation& location, const char* reason) { #ifndef NDEBUG - if (location.Log() && trace_flag_ != nullptr && trace_flag_->enabled()) { - const RefCount::Value old_refs = get(); + if (trace_flag_ != nullptr && trace_flag_->enabled()) { + const Value prior = get(); gpr_log(GPR_INFO, "%s:%p %s:%d ref_if_non_zero " "%" PRIdPTR " -> %" PRIdPTR " %s", trace_flag_->name(), this, location.file(), location.line(), - old_refs, old_refs + 1, reason); + prior, prior + 1, reason); } #endif - return RefIfNonZero(); + return value_.IncrementIfNonzero(); } // Decrements the ref-count and returns true if the ref-count reaches 0. bool Unref() { +#ifndef NDEBUG + // Grab a copy of the trace flag before the atomic change, since we + // can't safely access it afterwards if we're going to be freed. + auto* trace_flag = trace_flag_; +#endif const Value prior = value_.FetchSub(1, MemoryOrder::ACQ_REL); +#ifndef NDEBUG + if (trace_flag != nullptr && trace_flag->enabled()) { + gpr_log(GPR_INFO, "%s:%p unref %" PRIdPTR " -> %" PRIdPTR, + trace_flag->name(), this, prior, prior - 1); + } GPR_DEBUG_ASSERT(prior > 0); +#endif return prior == 1; } bool Unref(const DebugLocation& location, const char* reason) { #ifndef NDEBUG - if (location.Log() && trace_flag_ != nullptr && trace_flag_->enabled()) { - const RefCount::Value old_refs = get(); + // Grab a copy of the trace flag before the atomic change, since we + // can't safely access it afterwards if we're going to be freed. + auto* trace_flag = trace_flag_; +#endif + const Value prior = value_.FetchSub(1, MemoryOrder::ACQ_REL); +#ifndef NDEBUG + if (trace_flag != nullptr && trace_flag->enabled()) { gpr_log(GPR_INFO, "%s:%p %s:%d unref %" PRIdPTR " -> %" PRIdPTR " %s", - trace_flag_->name(), this, location.file(), location.line(), - old_refs, old_refs - 1, reason); + trace_flag->name(), this, location.file(), location.line(), prior, + prior - 1, reason); } + GPR_DEBUG_ASSERT(prior > 0); #endif - return Unref(); + return prior == 1; } private: From ba761f77c55cd2046844f11259fc32bd42355248 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 12 Jul 2019 15:21:11 -0700 Subject: [PATCH 50/55] Yapf. Pylint. Long-form birth certificate. --- tools/release/verify_python_release.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/release/verify_python_release.py b/tools/release/verify_python_release.py index af2892b61a5..6cbc39982a9 100644 --- a/tools/release/verify_python_release.py +++ b/tools/release/verify_python_release.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Verifies that all gRPC Python artifacts have been successfully published. This script is intended to be run from a directory containing the artifacts @@ -107,8 +106,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( "Verify a release. Run this from a directory containing only the" "artifacts to be uploaded. Note that PyPI may take several minutes" - "after the upload to reflect the proper metadata." - ) + "after the upload to reflect the proper metadata.") parser.add_argument("version") parser.add_argument( "packages", nargs='*', type=str, default=_DEFAULT_PACKAGES) From fa4e0f040faa48ec31b97f4778d71f94b73427f8 Mon Sep 17 00:00:00 2001 From: Juanli Shen Date: Fri, 12 Jul 2019 17:45:04 -0700 Subject: [PATCH 51/55] Fix segv in pick_first --- .../filters/client_channel/lb_policy/pick_first/pick_first.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index e0deed27442..411d528354a 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -176,7 +176,7 @@ void PickFirst::ExitIdleLocked() { } void PickFirst::ResetBackoffLocked() { - subchannel_list_->ResetBackoffLocked(); + if (subchannel_list_ != nullptr) subchannel_list_->ResetBackoffLocked(); if (latest_pending_subchannel_list_ != nullptr) { latest_pending_subchannel_list_->ResetBackoffLocked(); } From c556a02024a0922a51edb4976343dd6814515f5d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 15 Jul 2019 11:20:17 -0400 Subject: [PATCH 52/55] add per-rpc interop tests to managed grpc-dotnet client --- tools/run_tests/run_interop_tests.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index e71d308dcbe..d35723c917e 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -208,7 +208,10 @@ class AspNetCoreLanguage: def unimplemented_test_cases(self): return _SKIP_COMPRESSION + \ - _AUTH_TEST_CASES + ['compute_engine_creds'] + \ + ['jwt_token_creds'] + \ + _SKIP_GOOGLE_DEFAULT_CREDS + \ + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -821,8 +824,8 @@ def auth_options(language, test_case, google_default_creds_use_key_file, if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: if language in [ - 'csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', - 'ruby', 'nodepurejs' + 'csharp', 'csharpcoreclr', 'aspnetcore', 'node', 'php', 'php7', + 'python', 'ruby', 'nodepurejs' ]: env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file else: From 225a878e9f358de573c4e05f119bf106476ed19c Mon Sep 17 00:00:00 2001 From: Hope Casey-Allen Date: Thu, 11 Jul 2019 18:51:17 -0700 Subject: [PATCH 53/55] Track channelz server sockets in the server node --- src/core/lib/channel/channelz.cc | 33 +++++++++++++++++++++----------- src/core/lib/channel/channelz.h | 11 ++++++++--- src/core/lib/surface/server.cc | 32 +++++++++++++++---------------- src/core/lib/surface/server.h | 8 ++------ 4 files changed, 47 insertions(+), 37 deletions(-) diff --git a/src/core/lib/channel/channelz.cc b/src/core/lib/channel/channelz.cc index 184ba17889d..b9f65870cfe 100644 --- a/src/core/lib/channel/channelz.cc +++ b/src/core/lib/channel/channelz.cc @@ -309,31 +309,42 @@ ServerNode::ServerNode(grpc_server* server, size_t channel_tracer_max_nodes) ServerNode::~ServerNode() {} +void ServerNode::AddChildSocket(RefCountedPtr node) { + MutexLock lock(&child_mu_); + child_sockets_.insert(MakePair(node->uuid(), std::move(node))); +} + +void ServerNode::RemoveChildSocket(intptr_t child_uuid) { + MutexLock lock(&child_mu_); + child_sockets_.erase(child_uuid); +} + char* ServerNode::RenderServerSockets(intptr_t start_socket_id, intptr_t max_results) { - // if user does not set max_results, we choose 500. + // If user does not set max_results, we choose 500. size_t pagination_limit = max_results == 0 ? 500 : max_results; grpc_json* top_level_json = grpc_json_create(GRPC_JSON_OBJECT); grpc_json* json = top_level_json; grpc_json* json_iterator = nullptr; - ChildSocketsList socket_refs; - grpc_server_populate_server_sockets(server_, &socket_refs, start_socket_id); - // declared early so it can be used outside of the loop. - size_t i = 0; - if (!socket_refs.empty()) { - // create list of socket refs + MutexLock lock(&child_mu_); + size_t sockets_rendered = 0; + if (!child_sockets_.empty()) { + // Create list of socket refs grpc_json* array_parent = grpc_json_create_child( nullptr, json, "socketRef", nullptr, GRPC_JSON_ARRAY, false); - for (i = 0; i < GPR_MIN(socket_refs.size(), pagination_limit); ++i) { + const size_t limit = GPR_MIN(child_sockets_.size(), pagination_limit); + for (auto it = child_sockets_.lower_bound(start_socket_id); + it != child_sockets_.end() && sockets_rendered < limit; + ++it, ++sockets_rendered) { grpc_json* socket_ref_json = grpc_json_create_child( nullptr, array_parent, nullptr, nullptr, GRPC_JSON_OBJECT, false); json_iterator = grpc_json_add_number_string_child( - socket_ref_json, nullptr, "socketId", socket_refs[i]->uuid()); + socket_ref_json, nullptr, "socketId", it->first); grpc_json_create_child(json_iterator, socket_ref_json, "name", - socket_refs[i]->remote(), GRPC_JSON_STRING, false); + it->second->remote(), GRPC_JSON_STRING, false); } } - if (i == socket_refs.size()) { + if (sockets_rendered == child_sockets_.size()) { json_iterator = grpc_json_create_child(nullptr, json, "end", nullptr, GRPC_JSON_TRUE, false); } diff --git a/src/core/lib/channel/channelz.h b/src/core/lib/channel/channelz.h index d26896262ec..a02cb82cc75 100644 --- a/src/core/lib/channel/channelz.h +++ b/src/core/lib/channel/channelz.h @@ -64,7 +64,6 @@ intptr_t GetParentUuidFromArgs(const grpc_channel_args& args); typedef InlinedVector ChildRefsList; class SocketNode; -typedef InlinedVector, 10> ChildSocketsList; namespace testing { class CallCountingHelperPeer; @@ -207,12 +206,16 @@ class ChannelNode : public BaseNode { class ServerNode : public BaseNode { public: ServerNode(grpc_server* server, size_t channel_tracer_max_nodes); + ~ServerNode() override; grpc_json* RenderJson() override; - char* RenderServerSockets(intptr_t start_socket_id, - intptr_t pagination_limit); + char* RenderServerSockets(intptr_t start_socket_id, intptr_t max_results); + + void AddChildSocket(RefCountedPtr); + + void RemoveChildSocket(intptr_t child_uuid); // proxy methods to composed classes. void AddTraceEvent(ChannelTrace::Severity severity, const grpc_slice& data) { @@ -232,6 +235,8 @@ class ServerNode : public BaseNode { grpc_server* server_; CallCountingHelper call_counter_; ChannelTrace trace_; + Mutex child_mu_; // Guards child map below. + Map> child_sockets_; }; // Handles channelz bookkeeping for sockets diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index 6cd86003f01..a1c7d132ade 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -31,6 +31,7 @@ #include #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channelz.h" #include "src/core/lib/channel/connected_channel.h" #include "src/core/lib/debug/stats.h" #include "src/core/lib/gpr/mpscq.h" @@ -111,7 +112,7 @@ struct channel_data { uint32_t registered_method_max_probes; grpc_closure finish_destroy_channel_closure; grpc_closure channel_connectivity_changed; - grpc_core::RefCountedPtr socket_node; + intptr_t channelz_socket_uuid; }; typedef struct shutdown_tag { @@ -941,7 +942,6 @@ static grpc_error* init_channel_elem(grpc_channel_element* elem, static void destroy_channel_elem(grpc_channel_element* elem) { size_t i; channel_data* chand = static_cast(elem->channel_data); - chand->socket_node.reset(); if (chand->registered_methods) { for (i = 0; i < chand->registered_method_slots; i++) { grpc_slice_unref_internal(chand->registered_methods[i].method); @@ -952,6 +952,11 @@ static void destroy_channel_elem(grpc_channel_element* elem) { gpr_free(chand->registered_methods); } if (chand->server) { + if (chand->server->channelz_server != nullptr && + chand->channelz_socket_uuid != 0) { + chand->server->channelz_server->RemoveChildSocket( + chand->channelz_socket_uuid); + } gpr_mu_lock(&chand->server->mu_global); chand->next->prev = chand->prev; chand->prev->next = chand->next; @@ -1144,7 +1149,8 @@ void grpc_server_get_pollsets(grpc_server* server, grpc_pollset*** pollsets, void grpc_server_setup_transport( grpc_server* s, grpc_transport* transport, grpc_pollset* accepting_pollset, const grpc_channel_args* args, - grpc_core::RefCountedPtr socket_node, + const grpc_core::RefCountedPtr& + socket_node, grpc_resource_user* resource_user) { size_t num_registered_methods; size_t alloc; @@ -1166,7 +1172,12 @@ void grpc_server_setup_transport( chand->server = s; server_ref(s); chand->channel = channel; - chand->socket_node = std::move(socket_node); + if (socket_node != nullptr) { + chand->channelz_socket_uuid = socket_node->uuid(); + s->channelz_server->AddChildSocket(socket_node); + } else { + chand->channelz_socket_uuid = 0; + } size_t cq_idx; for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) { @@ -1241,19 +1252,6 @@ void grpc_server_setup_transport( grpc_transport_perform_op(transport, op); } -void grpc_server_populate_server_sockets( - grpc_server* s, grpc_core::channelz::ChildSocketsList* server_sockets, - intptr_t start_idx) { - gpr_mu_lock(&s->mu_global); - channel_data* c = nullptr; - for (c = s->root_channel_data.next; c != &s->root_channel_data; c = c->next) { - if (c->socket_node != nullptr && c->socket_node->uuid() >= start_idx) { - server_sockets->push_back(c->socket_node); - } - } - gpr_mu_unlock(&s->mu_global); -} - void grpc_server_populate_listen_sockets( grpc_server* server, grpc_core::channelz::ChildRefsList* listen_sockets) { gpr_mu_lock(&server->mu_global); diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h index 393bb242148..926a5825006 100644 --- a/src/core/lib/surface/server.h +++ b/src/core/lib/surface/server.h @@ -47,14 +47,10 @@ void grpc_server_add_listener(grpc_server* server, void* listener, void grpc_server_setup_transport( grpc_server* server, grpc_transport* transport, grpc_pollset* accepting_pollset, const grpc_channel_args* args, - grpc_core::RefCountedPtr socket_node, + const grpc_core::RefCountedPtr& + socket_node, grpc_resource_user* resource_user = nullptr); -/* fills in the uuids of all sockets used for connections on this server */ -void grpc_server_populate_server_sockets( - grpc_server* server, grpc_core::channelz::ChildSocketsList* server_sockets, - intptr_t start_idx); - /* fills in the uuids of all listen sockets on this server */ void grpc_server_populate_listen_sockets( grpc_server* server, grpc_core::channelz::ChildRefsList* listen_sockets); From b1d73a01f15ef7de1aa3da25c2a62be0b97f2087 Mon Sep 17 00:00:00 2001 From: Arjun Roy Date: Mon, 24 Jun 2019 15:15:26 -0700 Subject: [PATCH 54/55] Removed duplicate static table from hpack table. Removed an or instruction for every usage of static grpc metadata. Inlined hpack table lookups for static metadata. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This leads to faster hpack parser creation: BM_HpackParserInitDestroy 5.32µs ± 1% 0.06µs ± 1% -98.91% (p=0.000 n=18+19) And slightly faster parsing: BM_HpackParserParseHeader 456ns ± 1% 435ns ± 1% -4.74% (p=0.000 n=18+19) BM_HpackParserParseHeader 1.06µs ± 2% 1.04µs ± 2% -1.82% (p=0.000 n=19+20) It also yields a slight (0.5 - 1.0 microsecond) reduction in CPU time for fullstack unary pingpong: BM_UnaryPingPong/0/512 [polls/iter:3.0001 ] 23.9µs ± 2% 23.0µs ± 1% -3.63% (p=0.002 n=6+6) BM_UnaryPingPong/0/32768 [polls/iter:3.00015 ] 35.1µs ± 1% 34.2µs ± 1% -2.57% (p=0.036 n=5+3) BM_UnaryPingPong/8/0 [polls/iter:3.00011 ] 21.7µs ± 3% 21.2µs ± 2% -2.44% (p=0.017 n=6+5) --- .../chttp2/transport/hpack_parser.cc | 1 - .../transport/chttp2/transport/hpack_table.cc | 173 +------ .../transport/chttp2/transport/hpack_table.h | 56 ++- src/core/lib/transport/static_metadata.cc | 441 ++++++++++++++++++ src/core/lib/transport/static_metadata.h | 353 ++++---------- .../transport/chttp2/hpack_parser_test.cc | 4 + .../core/transport/chttp2/hpack_table_test.cc | 4 - test/cpp/microbenchmarks/bm_chttp2_hpack.cc | 7 + tools/codegen/core/gen_static_metadata.py | 32 +- 9 files changed, 627 insertions(+), 444 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.cc b/src/core/ext/transport/chttp2/transport/hpack_parser.cc index efbd997e994..616d6c56148 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.cc @@ -1559,7 +1559,6 @@ void grpc_chttp2_hpack_parser_init(grpc_chttp2_hpack_parser* p) { p->value.data.copied.length = 0; p->dynamic_table_update_allowed = 2; p->last_error = GRPC_ERROR_NONE; - grpc_chttp2_hptbl_init(&p->table); } void grpc_chttp2_hpack_parser_set_has_priority(grpc_chttp2_hpack_parser* p) { diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.cc b/src/core/ext/transport/chttp2/transport/hpack_table.cc index 9d1ac4b370f..f86332c6bc3 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_table.cc @@ -35,179 +35,18 @@ extern grpc_core::TraceFlag grpc_http_trace; -static struct { - const char* key; - const char* value; -} static_table[] = { - /* 0: */ - {nullptr, nullptr}, - /* 1: */ - {":authority", ""}, - /* 2: */ - {":method", "GET"}, - /* 3: */ - {":method", "POST"}, - /* 4: */ - {":path", "/"}, - /* 5: */ - {":path", "/index.html"}, - /* 6: */ - {":scheme", "http"}, - /* 7: */ - {":scheme", "https"}, - /* 8: */ - {":status", "200"}, - /* 9: */ - {":status", "204"}, - /* 10: */ - {":status", "206"}, - /* 11: */ - {":status", "304"}, - /* 12: */ - {":status", "400"}, - /* 13: */ - {":status", "404"}, - /* 14: */ - {":status", "500"}, - /* 15: */ - {"accept-charset", ""}, - /* 16: */ - {"accept-encoding", "gzip, deflate"}, - /* 17: */ - {"accept-language", ""}, - /* 18: */ - {"accept-ranges", ""}, - /* 19: */ - {"accept", ""}, - /* 20: */ - {"access-control-allow-origin", ""}, - /* 21: */ - {"age", ""}, - /* 22: */ - {"allow", ""}, - /* 23: */ - {"authorization", ""}, - /* 24: */ - {"cache-control", ""}, - /* 25: */ - {"content-disposition", ""}, - /* 26: */ - {"content-encoding", ""}, - /* 27: */ - {"content-language", ""}, - /* 28: */ - {"content-length", ""}, - /* 29: */ - {"content-location", ""}, - /* 30: */ - {"content-range", ""}, - /* 31: */ - {"content-type", ""}, - /* 32: */ - {"cookie", ""}, - /* 33: */ - {"date", ""}, - /* 34: */ - {"etag", ""}, - /* 35: */ - {"expect", ""}, - /* 36: */ - {"expires", ""}, - /* 37: */ - {"from", ""}, - /* 38: */ - {"host", ""}, - /* 39: */ - {"if-match", ""}, - /* 40: */ - {"if-modified-since", ""}, - /* 41: */ - {"if-none-match", ""}, - /* 42: */ - {"if-range", ""}, - /* 43: */ - {"if-unmodified-since", ""}, - /* 44: */ - {"last-modified", ""}, - /* 45: */ - {"link", ""}, - /* 46: */ - {"location", ""}, - /* 47: */ - {"max-forwards", ""}, - /* 48: */ - {"proxy-authenticate", ""}, - /* 49: */ - {"proxy-authorization", ""}, - /* 50: */ - {"range", ""}, - /* 51: */ - {"referer", ""}, - /* 52: */ - {"refresh", ""}, - /* 53: */ - {"retry-after", ""}, - /* 54: */ - {"server", ""}, - /* 55: */ - {"set-cookie", ""}, - /* 56: */ - {"strict-transport-security", ""}, - /* 57: */ - {"transfer-encoding", ""}, - /* 58: */ - {"user-agent", ""}, - /* 59: */ - {"vary", ""}, - /* 60: */ - {"via", ""}, - /* 61: */ - {"www-authenticate", ""}, -}; - -static uint32_t entries_for_bytes(uint32_t bytes) { - return (bytes + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - 1) / - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; -} - -void grpc_chttp2_hptbl_init(grpc_chttp2_hptbl* tbl) { - size_t i; - - memset(tbl, 0, sizeof(*tbl)); - tbl->current_table_bytes = tbl->max_bytes = - GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE; - tbl->max_entries = tbl->cap_entries = - entries_for_bytes(tbl->current_table_bytes); - tbl->ents = static_cast( - gpr_malloc(sizeof(*tbl->ents) * tbl->cap_entries)); - memset(tbl->ents, 0, sizeof(*tbl->ents) * tbl->cap_entries); - for (i = 1; i <= GRPC_CHTTP2_LAST_STATIC_ENTRY; i++) { - tbl->static_ents[i - 1] = grpc_mdelem_from_slices( - grpc_slice_intern( - grpc_slice_from_static_string_internal(static_table[i].key)), - grpc_slice_intern( - grpc_slice_from_static_string_internal(static_table[i].value))); - } -} - void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl* tbl) { size_t i; - for (i = 0; i < GRPC_CHTTP2_LAST_STATIC_ENTRY; i++) { - GRPC_MDELEM_UNREF(tbl->static_ents[i]); - } for (i = 0; i < tbl->num_ents; i++) { GRPC_MDELEM_UNREF(tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]); } gpr_free(tbl->ents); + tbl->ents = nullptr; } -grpc_mdelem grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl* tbl, - uint32_t tbl_index) { - /* Static table comes first, just return an entry from it */ - if (tbl_index <= GRPC_CHTTP2_LAST_STATIC_ENTRY) { - return tbl->static_ents[tbl_index - 1]; - } - /* Otherwise, find the value in the list of valid entries */ +grpc_mdelem grpc_chttp2_hptbl_lookup_dynamic_index(const grpc_chttp2_hptbl* tbl, + uint32_t tbl_index) { + /* Not static - find the value in the list of valid entries */ tbl_index -= (GRPC_CHTTP2_LAST_STATIC_ENTRY + 1); if (tbl_index < tbl->num_ents) { uint32_t offset = @@ -280,7 +119,7 @@ grpc_error* grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl* tbl, evict1(tbl); } tbl->current_table_bytes = bytes; - tbl->max_entries = entries_for_bytes(bytes); + tbl->max_entries = grpc_chttp2_hptbl::entries_for_bytes(bytes); if (tbl->max_entries > tbl->cap_entries) { rebuild_ents(tbl, GPR_MAX(tbl->max_entries, 2 * tbl->cap_entries)); } else if (tbl->max_entries < tbl->cap_entries / 3) { @@ -350,7 +189,7 @@ grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( /* See if the string is in the static table */ for (i = 0; i < GRPC_CHTTP2_LAST_STATIC_ENTRY; i++) { - grpc_mdelem ent = tbl->static_ents[i]; + grpc_mdelem ent = grpc_static_mdelem_manifested[i]; if (!grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDKEY(ent))) continue; r.index = i + 1u; r.has_value = grpc_slice_eq(GRPC_MDVALUE(md), GRPC_MDVALUE(ent)); diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.h b/src/core/ext/transport/chttp2/transport/hpack_table.h index 38f8bdda6ae..699b7eccaf2 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.h +++ b/src/core/ext/transport/chttp2/transport/hpack_table.h @@ -22,6 +22,7 @@ #include #include +#include "src/core/lib/gprpp/memory.h" #include "src/core/lib/iomgr/error.h" #include "src/core/lib/transport/metadata.h" #include "src/core/lib/transport/static_metadata.h" @@ -46,32 +47,45 @@ #endif /* hpack decoder table */ -typedef struct { +struct grpc_chttp2_hptbl { + static uint32_t entries_for_bytes(uint32_t bytes) { + return (bytes + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - 1) / + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; + } + static constexpr uint32_t kInitialCapacity = + (GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - + 1) / + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; + + grpc_chttp2_hptbl() { + GPR_DEBUG_ASSERT(!ents); + constexpr uint32_t AllocSize = sizeof(*ents) * kInitialCapacity; + ents = static_cast(gpr_malloc(AllocSize)); + memset(ents, 0, AllocSize); + } + /* the first used entry in ents */ - uint32_t first_ent; + uint32_t first_ent = 0; /* how many entries are in the table */ - uint32_t num_ents; + uint32_t num_ents = 0; /* the amount of memory used by the table, according to the hpack algorithm */ - uint32_t mem_used; + uint32_t mem_used = 0; /* the max memory allowed to be used by the table, according to the hpack algorithm */ - uint32_t max_bytes; + uint32_t max_bytes = GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE; /* the currently agreed size of the table, according to the hpack algorithm */ - uint32_t current_table_bytes; + uint32_t current_table_bytes = GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE; /* Maximum number of entries we could possibly fit in the table, given defined overheads */ - uint32_t max_entries; + uint32_t max_entries = kInitialCapacity; /* Number of entries allocated in ents */ - uint32_t cap_entries; + uint32_t cap_entries = kInitialCapacity; /* a circular buffer of headers - this is stored in the opposite order to what hpack specifies, in order to simplify table management a little... meaning lookups need to SUBTRACT from the end position */ - grpc_mdelem* ents; - grpc_mdelem static_ents[GRPC_CHTTP2_LAST_STATIC_ENTRY]; -} grpc_chttp2_hptbl; + grpc_mdelem* ents = nullptr; +}; -/* initialize a hpack table */ -void grpc_chttp2_hptbl_init(grpc_chttp2_hptbl* tbl); void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl* tbl); void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl* tbl, uint32_t max_bytes); @@ -79,8 +93,20 @@ grpc_error* grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl* tbl, uint32_t bytes); /* lookup a table entry based on its hpack index */ -grpc_mdelem grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl* tbl, - uint32_t index); +grpc_mdelem grpc_chttp2_hptbl_lookup_dynamic_index(const grpc_chttp2_hptbl* tbl, + uint32_t tbl_index); +inline grpc_mdelem grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl* tbl, + uint32_t index) { + /* Static table comes first, just return an entry from it. + NB: This imposes the constraint that the first + GRPC_CHTTP2_LAST_STATIC_ENTRY entries in the core static metadata table + must follow the hpack standard. If that changes, we *must* not rely on + reading the core static metadata table here; at that point we'd need our + own singleton static metadata in the correct order. */ + return index <= GRPC_CHTTP2_LAST_STATIC_ENTRY + ? grpc_static_mdelem_manifested[index - 1] + : grpc_chttp2_hptbl_lookup_dynamic_index(tbl, index); +} /* add a table entry to the index */ grpc_error* grpc_chttp2_hptbl_add(grpc_chttp2_hptbl* tbl, grpc_mdelem md) GRPC_MUST_USE_RESULT; diff --git a/src/core/lib/transport/static_metadata.cc b/src/core/lib/transport/static_metadata.cc index 5887e3515fb..8b0a6f482d7 100644 --- a/src/core/lib/transport/static_metadata.cc +++ b/src/core/lib/transport/static_metadata.cc @@ -337,6 +337,447 @@ const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { {&grpc_static_metadata_refcounts[106], {{21, g_bytes + 1234}}}, }; +/* Warning: the core static metadata currently operates under the soft +constraint that the first GRPC_CHTTP2_LAST_STATIC_ENTRY (61) entries must +contain metadata specified by the http2 hpack standard. The CHTTP2 transport +reads the core metadata with this assumption in mind. If the order of the core +static metadata is to be changed, then the CHTTP2 transport must be changed as +well to stop relying on the core metadata. */ + +grpc_mdelem grpc_static_mdelem_manifested[GRPC_STATIC_MDELEM_COUNT] = { + // clang-format off + /* GRPC_MDELEM_AUTHORITY_EMPTY: + ":authority": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[0].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_METHOD_GET: + ":method": "GET" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[1].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_METHOD_POST: + ":method": "POST" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[2].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_PATH_SLASH: + ":path": "/" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[3].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML: + ":path": "/index.html" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[4].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_SCHEME_HTTP: + ":scheme": "http" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[5].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_SCHEME_HTTPS: + ":scheme": "https" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[6].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_STATUS_200: + ":status": "200" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[7].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_STATUS_204: + ":status": "204" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[8].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_STATUS_206: + ":status": "206" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[9].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_STATUS_304: + ":status": "304" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[10].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_STATUS_400: + ":status": "400" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[11].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_STATUS_404: + ":status": "404" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[12].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_STATUS_500: + ":status": "500" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[13].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCEPT_CHARSET_EMPTY: + "accept-charset": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[14].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCEPT_ENCODING_GZIP_COMMA_DEFLATE: + "accept-encoding": "gzip, deflate" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[15].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCEPT_LANGUAGE_EMPTY: + "accept-language": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[16].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCEPT_RANGES_EMPTY: + "accept-ranges": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[17].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCEPT_EMPTY: + "accept": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[18].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCESS_CONTROL_ALLOW_ORIGIN_EMPTY: + "access-control-allow-origin": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[19].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_AGE_EMPTY: + "age": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[20].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ALLOW_EMPTY: + "allow": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[21].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_AUTHORIZATION_EMPTY: + "authorization": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[22].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CACHE_CONTROL_EMPTY: + "cache-control": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[23].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_DISPOSITION_EMPTY: + "content-disposition": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[24].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_ENCODING_EMPTY: + "content-encoding": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[25].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_LANGUAGE_EMPTY: + "content-language": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[26].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_LENGTH_EMPTY: + "content-length": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[27].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_LOCATION_EMPTY: + "content-location": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[28].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_RANGE_EMPTY: + "content-range": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[29].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_TYPE_EMPTY: + "content-type": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[30].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_COOKIE_EMPTY: + "cookie": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[31].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_DATE_EMPTY: + "date": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[32].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ETAG_EMPTY: + "etag": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[33].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_EXPECT_EMPTY: + "expect": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[34].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_EXPIRES_EMPTY: + "expires": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[35].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_FROM_EMPTY: + "from": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[36].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_HOST_EMPTY: + "host": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[37].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_IF_MATCH_EMPTY: + "if-match": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[38].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_IF_MODIFIED_SINCE_EMPTY: + "if-modified-since": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[39].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_IF_NONE_MATCH_EMPTY: + "if-none-match": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[40].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_IF_RANGE_EMPTY: + "if-range": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[41].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_IF_UNMODIFIED_SINCE_EMPTY: + "if-unmodified-since": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[42].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_LAST_MODIFIED_EMPTY: + "last-modified": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[43].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_LINK_EMPTY: + "link": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[44].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_LOCATION_EMPTY: + "location": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[45].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_MAX_FORWARDS_EMPTY: + "max-forwards": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[46].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY: + "proxy-authenticate": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[47].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY: + "proxy-authorization": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[48].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_RANGE_EMPTY: + "range": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[49].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_REFERER_EMPTY: + "referer": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[50].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_REFRESH_EMPTY: + "refresh": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[51].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_RETRY_AFTER_EMPTY: + "retry-after": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[52].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_SERVER_EMPTY: + "server": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[53].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_SET_COOKIE_EMPTY: + "set-cookie": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[54].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY: + "strict-transport-security": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[55].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_TRANSFER_ENCODING_EMPTY: + "transfer-encoding": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[56].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_USER_AGENT_EMPTY: + "user-agent": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[57].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_VARY_EMPTY: + "vary": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[58].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_VIA_EMPTY: + "via": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[59].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY: + "www-authenticate": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[60].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_STATUS_0: + "grpc-status": "0" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[61].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_STATUS_1: + "grpc-status": "1" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[62].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_STATUS_2: + "grpc-status": "2" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[63].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ENCODING_IDENTITY: + "grpc-encoding": "identity" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[64].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ENCODING_GZIP: + "grpc-encoding": "gzip" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[65].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ENCODING_DEFLATE: + "grpc-encoding": "deflate" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[66].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_TE_TRAILERS: + "te": "trailers" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[67].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC: + "content-type": "application/grpc" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[68].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_SCHEME_GRPC: + ":scheme": "grpc" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[69].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_METHOD_PUT: + ":method": "PUT" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[70].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCEPT_ENCODING_EMPTY: + "accept-encoding": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[71].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_ENCODING_IDENTITY: + "content-encoding": "identity" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[72].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_CONTENT_ENCODING_GZIP: + "content-encoding": "gzip" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[73].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_LB_TOKEN_EMPTY: + "lb-token": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[74].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_LB_COST_BIN_EMPTY: + "lb-cost-bin": "" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[75].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY: + "grpc-accept-encoding": "identity" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[76].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE: + "grpc-accept-encoding": "deflate" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[77].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE: + "grpc-accept-encoding": "identity,deflate" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[78].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ACCEPT_ENCODING_GZIP: + "grpc-accept-encoding": "gzip" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[79].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP: + "grpc-accept-encoding": "identity,gzip" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[80].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE_COMMA_GZIP: + "grpc-accept-encoding": "deflate,gzip" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[81].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE_COMMA_GZIP: + "grpc-accept-encoding": "identity,deflate,gzip" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[82].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY: + "accept-encoding": "identity" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[83].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCEPT_ENCODING_GZIP: + "accept-encoding": "gzip" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[84].data(), + GRPC_MDELEM_STORAGE_STATIC), + /* GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP: + "accept-encoding": "identity,gzip" */ + GRPC_MAKE_MDELEM( + &grpc_static_mdelem_table[85].data(), + GRPC_MDELEM_STORAGE_STATIC) + // clang-format on +}; uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h index 800ee04d6f9..458e87d30a7 100644 --- a/src/core/lib/transport/static_metadata.h +++ b/src/core/lib/transport/static_metadata.h @@ -272,350 +272,195 @@ extern grpc_slice_refcount extern grpc_core::StaticMetadata grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT]; extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; +extern grpc_mdelem grpc_static_mdelem_manifested[GRPC_STATIC_MDELEM_COUNT]; /* ":authority": "" */ -#define GRPC_MDELEM_AUTHORITY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[0].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_AUTHORITY_EMPTY (grpc_static_mdelem_manifested[0]) /* ":method": "GET" */ -#define GRPC_MDELEM_METHOD_GET \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[1].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_METHOD_GET (grpc_static_mdelem_manifested[1]) /* ":method": "POST" */ -#define GRPC_MDELEM_METHOD_POST \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[2].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_METHOD_POST (grpc_static_mdelem_manifested[2]) /* ":path": "/" */ -#define GRPC_MDELEM_PATH_SLASH \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[3].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_PATH_SLASH (grpc_static_mdelem_manifested[3]) /* ":path": "/index.html" */ -#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[4].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_PATH_SLASH_INDEX_DOT_HTML (grpc_static_mdelem_manifested[4]) /* ":scheme": "http" */ -#define GRPC_MDELEM_SCHEME_HTTP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[5].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_SCHEME_HTTP (grpc_static_mdelem_manifested[5]) /* ":scheme": "https" */ -#define GRPC_MDELEM_SCHEME_HTTPS \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[6].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_SCHEME_HTTPS (grpc_static_mdelem_manifested[6]) /* ":status": "200" */ -#define GRPC_MDELEM_STATUS_200 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[7].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_STATUS_200 (grpc_static_mdelem_manifested[7]) /* ":status": "204" */ -#define GRPC_MDELEM_STATUS_204 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[8].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_STATUS_204 (grpc_static_mdelem_manifested[8]) /* ":status": "206" */ -#define GRPC_MDELEM_STATUS_206 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[9].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_STATUS_206 (grpc_static_mdelem_manifested[9]) /* ":status": "304" */ -#define GRPC_MDELEM_STATUS_304 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[10].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_STATUS_304 (grpc_static_mdelem_manifested[10]) /* ":status": "400" */ -#define GRPC_MDELEM_STATUS_400 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[11].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_STATUS_400 (grpc_static_mdelem_manifested[11]) /* ":status": "404" */ -#define GRPC_MDELEM_STATUS_404 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[12].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_STATUS_404 (grpc_static_mdelem_manifested[12]) /* ":status": "500" */ -#define GRPC_MDELEM_STATUS_500 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[13].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_STATUS_500 (grpc_static_mdelem_manifested[13]) /* "accept-charset": "" */ -#define GRPC_MDELEM_ACCEPT_CHARSET_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[14].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCEPT_CHARSET_EMPTY (grpc_static_mdelem_manifested[14]) /* "accept-encoding": "gzip, deflate" */ -#define GRPC_MDELEM_ACCEPT_ENCODING_GZIP_COMMA_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[15].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCEPT_ENCODING_GZIP_COMMA_DEFLATE \ + (grpc_static_mdelem_manifested[15]) /* "accept-language": "" */ -#define GRPC_MDELEM_ACCEPT_LANGUAGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[16].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCEPT_LANGUAGE_EMPTY (grpc_static_mdelem_manifested[16]) /* "accept-ranges": "" */ -#define GRPC_MDELEM_ACCEPT_RANGES_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[17].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCEPT_RANGES_EMPTY (grpc_static_mdelem_manifested[17]) /* "accept": "" */ -#define GRPC_MDELEM_ACCEPT_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[18].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCEPT_EMPTY (grpc_static_mdelem_manifested[18]) /* "access-control-allow-origin": "" */ -#define GRPC_MDELEM_ACCESS_CONTROL_ALLOW_ORIGIN_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[19].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCESS_CONTROL_ALLOW_ORIGIN_EMPTY \ + (grpc_static_mdelem_manifested[19]) /* "age": "" */ -#define GRPC_MDELEM_AGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[20].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_AGE_EMPTY (grpc_static_mdelem_manifested[20]) /* "allow": "" */ -#define GRPC_MDELEM_ALLOW_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[21].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ALLOW_EMPTY (grpc_static_mdelem_manifested[21]) /* "authorization": "" */ -#define GRPC_MDELEM_AUTHORIZATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[22].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_AUTHORIZATION_EMPTY (grpc_static_mdelem_manifested[22]) /* "cache-control": "" */ -#define GRPC_MDELEM_CACHE_CONTROL_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[23].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CACHE_CONTROL_EMPTY (grpc_static_mdelem_manifested[23]) /* "content-disposition": "" */ -#define GRPC_MDELEM_CONTENT_DISPOSITION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[24].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_DISPOSITION_EMPTY \ + (grpc_static_mdelem_manifested[24]) /* "content-encoding": "" */ -#define GRPC_MDELEM_CONTENT_ENCODING_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[25].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_ENCODING_EMPTY (grpc_static_mdelem_manifested[25]) /* "content-language": "" */ -#define GRPC_MDELEM_CONTENT_LANGUAGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[26].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_LANGUAGE_EMPTY (grpc_static_mdelem_manifested[26]) /* "content-length": "" */ -#define GRPC_MDELEM_CONTENT_LENGTH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[27].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_LENGTH_EMPTY (grpc_static_mdelem_manifested[27]) /* "content-location": "" */ -#define GRPC_MDELEM_CONTENT_LOCATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[28].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_LOCATION_EMPTY (grpc_static_mdelem_manifested[28]) /* "content-range": "" */ -#define GRPC_MDELEM_CONTENT_RANGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[29].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_RANGE_EMPTY (grpc_static_mdelem_manifested[29]) /* "content-type": "" */ -#define GRPC_MDELEM_CONTENT_TYPE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[30].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_TYPE_EMPTY (grpc_static_mdelem_manifested[30]) /* "cookie": "" */ -#define GRPC_MDELEM_COOKIE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[31].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_COOKIE_EMPTY (grpc_static_mdelem_manifested[31]) /* "date": "" */ -#define GRPC_MDELEM_DATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[32].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_DATE_EMPTY (grpc_static_mdelem_manifested[32]) /* "etag": "" */ -#define GRPC_MDELEM_ETAG_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[33].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ETAG_EMPTY (grpc_static_mdelem_manifested[33]) /* "expect": "" */ -#define GRPC_MDELEM_EXPECT_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[34].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_EXPECT_EMPTY (grpc_static_mdelem_manifested[34]) /* "expires": "" */ -#define GRPC_MDELEM_EXPIRES_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[35].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_EXPIRES_EMPTY (grpc_static_mdelem_manifested[35]) /* "from": "" */ -#define GRPC_MDELEM_FROM_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[36].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_FROM_EMPTY (grpc_static_mdelem_manifested[36]) /* "host": "" */ -#define GRPC_MDELEM_HOST_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[37].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_HOST_EMPTY (grpc_static_mdelem_manifested[37]) /* "if-match": "" */ -#define GRPC_MDELEM_IF_MATCH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[38].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_IF_MATCH_EMPTY (grpc_static_mdelem_manifested[38]) /* "if-modified-since": "" */ -#define GRPC_MDELEM_IF_MODIFIED_SINCE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[39].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_IF_MODIFIED_SINCE_EMPTY (grpc_static_mdelem_manifested[39]) /* "if-none-match": "" */ -#define GRPC_MDELEM_IF_NONE_MATCH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[40].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_IF_NONE_MATCH_EMPTY (grpc_static_mdelem_manifested[40]) /* "if-range": "" */ -#define GRPC_MDELEM_IF_RANGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[41].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_IF_RANGE_EMPTY (grpc_static_mdelem_manifested[41]) /* "if-unmodified-since": "" */ -#define GRPC_MDELEM_IF_UNMODIFIED_SINCE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[42].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_IF_UNMODIFIED_SINCE_EMPTY \ + (grpc_static_mdelem_manifested[42]) /* "last-modified": "" */ -#define GRPC_MDELEM_LAST_MODIFIED_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[43].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_LAST_MODIFIED_EMPTY (grpc_static_mdelem_manifested[43]) /* "link": "" */ -#define GRPC_MDELEM_LINK_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[44].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_LINK_EMPTY (grpc_static_mdelem_manifested[44]) /* "location": "" */ -#define GRPC_MDELEM_LOCATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[45].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_LOCATION_EMPTY (grpc_static_mdelem_manifested[45]) /* "max-forwards": "" */ -#define GRPC_MDELEM_MAX_FORWARDS_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[46].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_MAX_FORWARDS_EMPTY (grpc_static_mdelem_manifested[46]) /* "proxy-authenticate": "" */ -#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[47].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY (grpc_static_mdelem_manifested[47]) /* "proxy-authorization": "" */ -#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[48].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY \ + (grpc_static_mdelem_manifested[48]) /* "range": "" */ -#define GRPC_MDELEM_RANGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[49].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_RANGE_EMPTY (grpc_static_mdelem_manifested[49]) /* "referer": "" */ -#define GRPC_MDELEM_REFERER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[50].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_REFERER_EMPTY (grpc_static_mdelem_manifested[50]) /* "refresh": "" */ -#define GRPC_MDELEM_REFRESH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[51].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_REFRESH_EMPTY (grpc_static_mdelem_manifested[51]) /* "retry-after": "" */ -#define GRPC_MDELEM_RETRY_AFTER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[52].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_RETRY_AFTER_EMPTY (grpc_static_mdelem_manifested[52]) /* "server": "" */ -#define GRPC_MDELEM_SERVER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[53].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_SERVER_EMPTY (grpc_static_mdelem_manifested[53]) /* "set-cookie": "" */ -#define GRPC_MDELEM_SET_COOKIE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[54].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_SET_COOKIE_EMPTY (grpc_static_mdelem_manifested[54]) /* "strict-transport-security": "" */ -#define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[55].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \ + (grpc_static_mdelem_manifested[55]) /* "transfer-encoding": "" */ -#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY (grpc_static_mdelem_manifested[56]) /* "user-agent": "" */ -#define GRPC_MDELEM_USER_AGENT_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_USER_AGENT_EMPTY (grpc_static_mdelem_manifested[57]) /* "vary": "" */ -#define GRPC_MDELEM_VARY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_VARY_EMPTY (grpc_static_mdelem_manifested[58]) /* "via": "" */ -#define GRPC_MDELEM_VIA_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_VIA_EMPTY (grpc_static_mdelem_manifested[59]) /* "www-authenticate": "" */ -#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY (grpc_static_mdelem_manifested[60]) /* "grpc-status": "0" */ -#define GRPC_MDELEM_GRPC_STATUS_0 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_GRPC_STATUS_0 (grpc_static_mdelem_manifested[61]) /* "grpc-status": "1" */ -#define GRPC_MDELEM_GRPC_STATUS_1 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_GRPC_STATUS_1 (grpc_static_mdelem_manifested[62]) /* "grpc-status": "2" */ -#define GRPC_MDELEM_GRPC_STATUS_2 \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_GRPC_STATUS_2 (grpc_static_mdelem_manifested[63]) /* "grpc-encoding": "identity" */ -#define GRPC_MDELEM_GRPC_ENCODING_IDENTITY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_GRPC_ENCODING_IDENTITY (grpc_static_mdelem_manifested[64]) /* "grpc-encoding": "gzip" */ -#define GRPC_MDELEM_GRPC_ENCODING_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_GRPC_ENCODING_GZIP (grpc_static_mdelem_manifested[65]) /* "grpc-encoding": "deflate" */ -#define GRPC_MDELEM_GRPC_ENCODING_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_GRPC_ENCODING_DEFLATE (grpc_static_mdelem_manifested[66]) /* "te": "trailers" */ -#define GRPC_MDELEM_TE_TRAILERS \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_TE_TRAILERS (grpc_static_mdelem_manifested[67]) /* "content-type": "application/grpc" */ -#define GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC \ + (grpc_static_mdelem_manifested[68]) /* ":scheme": "grpc" */ -#define GRPC_MDELEM_SCHEME_GRPC \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_SCHEME_GRPC (grpc_static_mdelem_manifested[69]) /* ":method": "PUT" */ -#define GRPC_MDELEM_METHOD_PUT \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_METHOD_PUT (grpc_static_mdelem_manifested[70]) /* "accept-encoding": "" */ -#define GRPC_MDELEM_ACCEPT_ENCODING_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCEPT_ENCODING_EMPTY (grpc_static_mdelem_manifested[71]) /* "content-encoding": "identity" */ -#define GRPC_MDELEM_CONTENT_ENCODING_IDENTITY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_ENCODING_IDENTITY \ + (grpc_static_mdelem_manifested[72]) /* "content-encoding": "gzip" */ -#define GRPC_MDELEM_CONTENT_ENCODING_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_CONTENT_ENCODING_GZIP (grpc_static_mdelem_manifested[73]) /* "lb-token": "" */ -#define GRPC_MDELEM_LB_TOKEN_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_LB_TOKEN_EMPTY (grpc_static_mdelem_manifested[74]) /* "lb-cost-bin": "" */ -#define GRPC_MDELEM_LB_COST_BIN_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_LB_COST_BIN_EMPTY (grpc_static_mdelem_manifested[75]) /* "grpc-accept-encoding": "identity" */ -#define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY \ + (grpc_static_mdelem_manifested[76]) /* "grpc-accept-encoding": "deflate" */ -#define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE \ + (grpc_static_mdelem_manifested[77]) /* "grpc-accept-encoding": "identity,deflate" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) + (grpc_static_mdelem_manifested[78]) /* "grpc-accept-encoding": "gzip" */ -#define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_GZIP \ + (grpc_static_mdelem_manifested[79]) /* "grpc-accept-encoding": "identity,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[80].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) + (grpc_static_mdelem_manifested[80]) /* "grpc-accept-encoding": "deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[81].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) + (grpc_static_mdelem_manifested[81]) /* "grpc-accept-encoding": "identity,deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[82].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) + (grpc_static_mdelem_manifested[82]) /* "accept-encoding": "identity" */ -#define GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[83].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY (grpc_static_mdelem_manifested[83]) /* "accept-encoding": "gzip" */ -#define GRPC_MDELEM_ACCEPT_ENCODING_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[84].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCEPT_ENCODING_GZIP (grpc_static_mdelem_manifested[84]) /* "accept-encoding": "identity,gzip" */ -#define GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[85].data(), \ - GRPC_MDELEM_STORAGE_STATIC)) +#define GRPC_MDELEM_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \ + (grpc_static_mdelem_manifested[85]) grpc_mdelem grpc_static_mdelem_for_static_strings(intptr_t a, intptr_t b); typedef enum { diff --git a/test/core/transport/chttp2/hpack_parser_test.cc b/test/core/transport/chttp2/hpack_parser_test.cc index 882ad726b5a..8be7dc17ed1 100644 --- a/test/core/transport/chttp2/hpack_parser_test.cc +++ b/test/core/transport/chttp2/hpack_parser_test.cc @@ -102,6 +102,7 @@ static void test_vectors(grpc_slice_split_mode mode) { grpc_chttp2_hpack_parser_destroy(&parser); grpc_chttp2_hpack_parser_init(&parser); + new (&parser.table) grpc_chttp2_hptbl(); /* D.3.1 */ test_vector(&parser, mode, "8286 8441 0f77 7777 2e65 7861 6d70 6c65" @@ -122,6 +123,7 @@ static void test_vectors(grpc_slice_split_mode mode) { grpc_chttp2_hpack_parser_destroy(&parser); grpc_chttp2_hpack_parser_init(&parser); + new (&parser.table) grpc_chttp2_hptbl(); /* D.4.1 */ test_vector(&parser, mode, "8286 8441 8cf1 e3c2 e5f2 3a6b a0ab 90f4" @@ -142,6 +144,7 @@ static void test_vectors(grpc_slice_split_mode mode) { grpc_chttp2_hpack_parser_destroy(&parser); grpc_chttp2_hpack_parser_init(&parser); + new (&parser.table) grpc_chttp2_hptbl(); grpc_chttp2_hptbl_set_max_bytes(&parser.table, 256); grpc_chttp2_hptbl_set_current_table_size(&parser.table, 256); /* D.5.1 */ @@ -176,6 +179,7 @@ static void test_vectors(grpc_slice_split_mode mode) { grpc_chttp2_hpack_parser_destroy(&parser); grpc_chttp2_hpack_parser_init(&parser); + new (&parser.table) grpc_chttp2_hptbl(); grpc_chttp2_hptbl_set_max_bytes(&parser.table, 256); grpc_chttp2_hptbl_set_current_table_size(&parser.table, 256); /* D.6.1 */ diff --git a/test/core/transport/chttp2/hpack_table_test.cc b/test/core/transport/chttp2/hpack_table_test.cc index c31975786f0..32b011b247b 100644 --- a/test/core/transport/chttp2/hpack_table_test.cc +++ b/test/core/transport/chttp2/hpack_table_test.cc @@ -48,8 +48,6 @@ static void test_static_lookup(void) { grpc_core::ExecCtx exec_ctx; grpc_chttp2_hptbl tbl; - grpc_chttp2_hptbl_init(&tbl); - LOG_TEST("test_static_lookup"); assert_index(&tbl, 1, ":authority", ""); assert_index(&tbl, 2, ":method", "GET"); @@ -125,7 +123,6 @@ static void test_many_additions(void) { LOG_TEST("test_many_additions"); grpc_core::ExecCtx exec_ctx; - grpc_chttp2_hptbl_init(&tbl); for (i = 0; i < 100000; i++) { grpc_mdelem elem; @@ -172,7 +169,6 @@ static void test_find(void) { LOG_TEST("test_find"); - grpc_chttp2_hptbl_init(&tbl); elem = grpc_mdelem_from_slices(grpc_slice_from_static_string("abc"), grpc_slice_from_static_string("xyz")); GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index d0ed1da7671..1d2ddf13f6a 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -433,8 +433,15 @@ static void BM_HpackParserInitDestroy(benchmark::State& state) { TrackCounters track_counters; grpc_core::ExecCtx exec_ctx; grpc_chttp2_hpack_parser p; + // Initial destruction so we don't leak memory in the loop. + grpc_chttp2_hptbl_destroy(&p.table); while (state.KeepRunning()) { grpc_chttp2_hpack_parser_init(&p); + // Note that grpc_chttp2_hpack_parser_destroy frees the table dynamic + // elements so we need to recreate it here. In actual operation, + // grpc_core::New allocates the table once + // and for all. + new (&p.table) grpc_chttp2_hptbl(); grpc_chttp2_hpack_parser_destroy(&p); grpc_core::ExecCtx::Get()->Flush(); } diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index a149e001d75..95a545f5542 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -451,11 +451,37 @@ print >> H, ('extern grpc_core::StaticMetadata ' 'grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT];') print >> H, ('extern uintptr_t ' 'grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];') +print >> H, ('extern grpc_mdelem ' + 'grpc_static_mdelem_manifested[GRPC_STATIC_MDELEM_COUNT];') +print >> C, (''' +/* Warning: the core static metadata currently operates under the soft constraint +that the first GRPC_CHTTP2_LAST_STATIC_ENTRY (61) entries must contain +metadata specified by the http2 hpack standard. The CHTTP2 transport reads the +core metadata with this assumption in mind. If the order of the core static +metadata is to be changed, then the CHTTP2 transport must be changed as well to +stop relying on the core metadata. */ +''') +print >> C, ('grpc_mdelem ' + 'grpc_static_mdelem_manifested[GRPC_STATIC_MDELEM_COUNT] = {') +print >> C, '// clang-format off' +static_mds = [] for i, elem in enumerate(all_elems): + md_name = mangle(elem).upper() + md_human_readable = '"%s": "%s"' % elem + md_spec = ' /* %s: \n %s */\n' % (md_name, md_human_readable) + md_spec += ' GRPC_MAKE_MDELEM(\n' + md_spec += ((' &grpc_static_mdelem_table[%d].data(),\n' % i) + + ' GRPC_MDELEM_STORAGE_STATIC)') + static_mds.append(md_spec) +print >> C, ',\n'.join(static_mds) +print >> C, '// clang-format on' +print >> C, ('};') + +for i, elem in enumerate(all_elems): + md_name = mangle(elem).upper() print >> H, '/* "%s": "%s" */' % elem - print >> H, ( - '#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d].data(), ' - 'GRPC_MDELEM_STORAGE_STATIC))') % (mangle(elem).upper(), i) + print >> H, ('#define %s (grpc_static_mdelem_manifested[%d])' % (md_name, + i)) print >> H print >> C, ('uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] ' From 666d73e39cfaa7f40082c7234ca4f111a2178bce Mon Sep 17 00:00:00 2001 From: Arjun Roy Date: Wed, 3 Jul 2019 14:30:25 -0700 Subject: [PATCH 55/55] Cached metadata lookup for hpack_parser. --- .../chttp2/transport/hpack_parser.cc | 64 +++++++++++++++++-- .../transport/chttp2/transport/hpack_parser.h | 8 +++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.cc b/src/core/ext/transport/chttp2/transport/hpack_parser.cc index efbd997e994..f83f67ed195 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.cc @@ -779,6 +779,7 @@ static grpc_error* parse_indexed_field(grpc_chttp2_hpack_parser* p, const uint8_t* cur, const uint8_t* end) { p->dynamic_table_update_allowed = 0; p->index = (*cur) & 0x7f; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ return finish_indexed_field(p, cur + 1, end); } @@ -791,17 +792,32 @@ static grpc_error* parse_indexed_field_x(grpc_chttp2_hpack_parser* p, p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = 0x7f; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ p->parsing.value = &p->index; return parse_value0(p, cur + 1, end); } +/* When finishing with a header, get the cached md element for this index. + This is set in parse_value_string(). We ensure (in debug mode) that the + cached metadata corresponds with the index we are examining. */ +static grpc_mdelem get_precomputed_md_for_idx(grpc_chttp2_hpack_parser* p) { + GPR_DEBUG_ASSERT(p->md_for_index.payload != 0); + GPR_DEBUG_ASSERT(static_cast(p->index) == p->precomputed_md_index); + grpc_mdelem md = p->md_for_index; + GPR_DEBUG_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ +#ifndef NDEBUG + p->precomputed_md_index = -1; +#endif + return md; +} + /* finish a literal header with incremental indexing */ static grpc_error* finish_lithdr_incidx(grpc_chttp2_hpack_parser* p, const uint8_t* cur, const uint8_t* end) { - grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index); - GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX(); + grpc_mdelem md = get_precomputed_md_for_idx(p); grpc_error* err = on_hdr( p, grpc_mdelem_from_slices(grpc_slice_ref_internal(GRPC_MDKEY(md)), take_string(p, &p->value, true))); @@ -829,6 +845,7 @@ static grpc_error* parse_lithdr_incidx(grpc_chttp2_hpack_parser* p, p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = (*cur) & 0x3f; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ return parse_string_prefix(p, cur + 1, end); } @@ -842,6 +859,7 @@ static grpc_error* parse_lithdr_incidx_x(grpc_chttp2_hpack_parser* p, p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = 0x3f; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ p->parsing.value = &p->index; return parse_value0(p, cur + 1, end); } @@ -862,9 +880,8 @@ static grpc_error* parse_lithdr_incidx_v(grpc_chttp2_hpack_parser* p, static grpc_error* finish_lithdr_notidx(grpc_chttp2_hpack_parser* p, const uint8_t* cur, const uint8_t* end) { - grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index); - GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX(); + grpc_mdelem md = get_precomputed_md_for_idx(p); grpc_error* err = on_hdr( p, grpc_mdelem_from_slices(grpc_slice_ref_internal(GRPC_MDKEY(md)), take_string(p, &p->value, false))); @@ -892,6 +909,7 @@ static grpc_error* parse_lithdr_notidx(grpc_chttp2_hpack_parser* p, p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = (*cur) & 0xf; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ return parse_string_prefix(p, cur + 1, end); } @@ -905,6 +923,7 @@ static grpc_error* parse_lithdr_notidx_x(grpc_chttp2_hpack_parser* p, p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = 0xf; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ p->parsing.value = &p->index; return parse_value0(p, cur + 1, end); } @@ -925,9 +944,8 @@ static grpc_error* parse_lithdr_notidx_v(grpc_chttp2_hpack_parser* p, static grpc_error* finish_lithdr_nvridx(grpc_chttp2_hpack_parser* p, const uint8_t* cur, const uint8_t* end) { - grpc_mdelem md = grpc_chttp2_hptbl_lookup(&p->table, p->index); - GPR_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX(); + grpc_mdelem md = get_precomputed_md_for_idx(p); grpc_error* err = on_hdr( p, grpc_mdelem_from_slices(grpc_slice_ref_internal(GRPC_MDKEY(md)), take_string(p, &p->value, false))); @@ -955,6 +973,7 @@ static grpc_error* parse_lithdr_nvridx(grpc_chttp2_hpack_parser* p, p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = (*cur) & 0xf; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ return parse_string_prefix(p, cur + 1, end); } @@ -968,6 +987,7 @@ static grpc_error* parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser* p, p->dynamic_table_update_allowed = 0; p->next_state = and_then; p->index = 0xf; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ p->parsing.value = &p->index; return parse_value0(p, cur + 1, end); } @@ -1007,6 +1027,7 @@ static grpc_error* parse_max_tbl_size(grpc_chttp2_hpack_parser* p, } p->dynamic_table_update_allowed--; p->index = (*cur) & 0x1f; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ return finish_max_tbl_size(p, cur + 1, end); } @@ -1025,6 +1046,7 @@ static grpc_error* parse_max_tbl_size_x(grpc_chttp2_hpack_parser* p, p->dynamic_table_update_allowed--; p->next_state = and_then; p->index = 0x1f; + p->md_for_index.payload = 0; /* Invalidate cached md when index changes. */ p->parsing.value = &p->index; return parse_value0(p, cur + 1, end); } @@ -1499,6 +1521,23 @@ static bool is_binary_literal_header(grpc_chttp2_hpack_parser* p) { : p->key.data.referenced); } +/* Cache the metadata for the given index during initial parsing. This avoids a + pointless recomputation of the metadata when finishing a header. We read the + cached value in get_precomputed_md_for_idx(). */ +static void set_precomputed_md_idx(grpc_chttp2_hpack_parser* p, + grpc_mdelem md) { + GPR_DEBUG_ASSERT(p->md_for_index.payload == 0); + GPR_DEBUG_ASSERT(p->precomputed_md_index == -1); + p->md_for_index = md; +#ifndef NDEBUG + p->precomputed_md_index = p->index; +#endif +} + +/* Determines if a metadata element key associated with the current parser index + is a binary indexed header during string parsing. We'll need to revisit this + metadata when we're done parsing, so we cache the metadata for this index + here using set_precomputed_md_idx(). */ static grpc_error* is_binary_indexed_header(grpc_chttp2_hpack_parser* p, bool* is) { grpc_mdelem elem = grpc_chttp2_hptbl_lookup(&p->table, p->index); @@ -1519,6 +1558,7 @@ static grpc_error* is_binary_indexed_header(grpc_chttp2_hpack_parser* p, * interned. * 4. Both static and interned element slices have non-null refcounts. */ *is = grpc_is_refcounted_slice_binary_header(GRPC_MDKEY(elem)); + set_precomputed_md_idx(p, elem); return GRPC_ERROR_NONE; } @@ -1557,6 +1597,18 @@ void grpc_chttp2_hpack_parser_init(grpc_chttp2_hpack_parser* p) { p->value.data.copied.str = nullptr; p->value.data.copied.capacity = 0; p->value.data.copied.length = 0; + /* Cached metadata for the current index the parser is handling. This is set + to 0 initially, invalidated when the index changes, and invalidated when it + is read (by get_precomputed_md_for_idx()). It is set during string parsing, + by set_precomputed_md_idx() - which is called by parse_value_string(). + The goal here is to avoid recomputing the metadata for the index when + finishing with a header as well as the initial parse. */ + p->md_for_index.payload = 0; +#ifndef NDEBUG + /* In debug mode, this ensures that the cached metadata we're reading is in + * fact correct for the index we are examining. */ + p->precomputed_md_index = -1; +#endif p->dynamic_table_update_allowed = 2; p->last_error = GRPC_ERROR_NONE; grpc_chttp2_hptbl_init(&p->table); diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.h b/src/core/ext/transport/chttp2/transport/hpack_parser.h index 3dc8e13bea2..c5691244028 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.h +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.h @@ -69,6 +69,14 @@ struct grpc_chttp2_hpack_parser { grpc_chttp2_hpack_parser_string value; /* parsed index */ uint32_t index; + /* When we parse a value string, we determine the metadata element for a + specific index, which we need again when we're finishing up with that + header. To avoid calculating the metadata element for that index a second + time at that stage, we cache (and invalidate) the element here. */ + grpc_mdelem md_for_index; +#ifndef NDEBUG + int64_t precomputed_md_index; +#endif /* length of source bytes for the currently parsing string */ uint32_t strlen; /* number of source bytes read for the currently parsing string */