From e1523e95c102a3eec48fef34350bca206c0a6546 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 14 Mar 2017 21:10:42 -0700 Subject: [PATCH 01/51] Track calls to gpr_now() --- src/core/lib/support/time_posix.c | 8 ++++++++ test/cpp/microbenchmarks/bm_call_create.cc | 2 ++ test/cpp/microbenchmarks/helpers.cc | 4 ++++ test/cpp/microbenchmarks/helpers.h | 3 +++ tools/profiling/microbenchmarks/bm2bq.py | 2 +- 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/core/lib/support/time_posix.c b/src/core/lib/support/time_posix.c index a69c501e9fb..9bfec7782a2 100644 --- a/src/core/lib/support/time_posix.c +++ b/src/core/lib/support/time_posix.c @@ -42,6 +42,7 @@ #ifdef __linux__ #include #endif +#include #include #include #include "src/core/lib/support/block_annotate.h" @@ -144,7 +145,14 @@ static gpr_timespec now_impl(gpr_clock_type clock) { gpr_timespec (*gpr_now_impl)(gpr_clock_type clock_type) = now_impl; +#ifdef GPR_LOW_LEVEL_COUNTERS +gpr_atm gpr_now_call_count; +#endif + gpr_timespec gpr_now(gpr_clock_type clock_type) { +#ifdef GPR_LOW_LEVEL_COUNTERS + __atomic_fetch_add(&gpr_now_call_count, 1, __ATOMIC_RELAXED); +#endif return gpr_now_impl(clock_type); } diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 014e2b96b55..146f8e6bad3 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -66,10 +66,12 @@ auto &force_library_initialization = Library::get(); void BM_Zalloc(benchmark::State &state) { // speed of light for call creation is zalloc, so benchmark a few interesting // sizes + TrackCounters track_counters; size_t sz = state.range(0); while (state.KeepRunning()) { gpr_free(gpr_zalloc(sz)); } + track_counters.Finish(state); } BENCHMARK(BM_Zalloc) ->Arg(64) diff --git a/test/cpp/microbenchmarks/helpers.cc b/test/cpp/microbenchmarks/helpers.cc index d277c5984cd..6550742453a 100644 --- a/test/cpp/microbenchmarks/helpers.cc +++ b/test/cpp/microbenchmarks/helpers.cc @@ -57,6 +57,10 @@ void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) { << ((double)(gpr_atm_no_barrier_load(&gpr_counter_atm_add) - atm_add_at_start_) / (double)state.iterations()) + << " nows/iter:" + << ((double)(gpr_atm_no_barrier_load(&gpr_now_call_count) - + now_calls_at_start_) / + (double)state.iterations()) << " allocs/iter:" << ((double)(counters_at_end.total_allocs_absolute - counters_at_start_.total_allocs_absolute) / diff --git a/test/cpp/microbenchmarks/helpers.h b/test/cpp/microbenchmarks/helpers.h index f44b7cf83a7..af401dd7bbe 100644 --- a/test/cpp/microbenchmarks/helpers.h +++ b/test/cpp/microbenchmarks/helpers.h @@ -72,6 +72,7 @@ class Library { extern "C" gpr_atm gpr_mu_locks; extern "C" gpr_atm gpr_counter_atm_cas; extern "C" gpr_atm gpr_counter_atm_add; +extern "C" gpr_atm gpr_now_call_count; #endif class TrackCounters { @@ -86,6 +87,8 @@ class TrackCounters { gpr_atm_no_barrier_load(&gpr_counter_atm_cas); const size_t atm_add_at_start_ = gpr_atm_no_barrier_load(&gpr_counter_atm_add); + const size_t now_calls_at_start_ = + gpr_atm_no_barrier_load(&gpr_now_call_count); grpc_memory_counters counters_at_start_ = grpc_memory_counters_snapshot(); #endif }; diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index ffb11f57d8f..99cf4a558cf 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -71,6 +71,7 @@ columns = [ ('end_of_stream', 'boolean'), ('header_bytes_per_iteration', 'float'), ('framing_bytes_per_iteration', 'float'), + ('nows_per_iteration', 'float'), ] SANITIZE = { @@ -103,4 +104,3 @@ for row in bm_json.expand_json(js, js2): if row[name] == '': continue sane_row[name] = SANITIZE[sql_type](row[name]) writer.writerow(sane_row) - From 1eb96dc7fd70d72849b638aafd39de815521b540 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 22 Mar 2017 12:19:03 -0700 Subject: [PATCH 02/51] Fix semantics of LB policy selection. --- doc/service_config.md | 20 ++++++++++++++------ src/core/ext/client_channel/client_channel.c | 18 ++++++++---------- src/core/ext/lb_policy/grpclb/grpclb.c | 8 ++++---- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/doc/service_config.md b/doc/service_config.md index ecc23817d12..c5d4ef42b19 100644 --- a/doc/service_config.md +++ b/doc/service_config.md @@ -13,12 +13,20 @@ The service config is a JSON string of the following form: ``` { // Load balancing policy name. - // Supported values are 'round_robin' and 'grpclb'. - // Optional; if unset, the default behavior is pick the first available - // backend. - // Note that if the resolver returns only balancer addresses and no - // backend addresses, gRPC will always use the 'grpclb' policy, - // regardless of what this field is set to. + // Currently, the only selectable client-side policy provided with gRPC + // is 'round_robin', but third parties may add their own policies. + // This field is optional; if unset, the default behavior is to pick + // the first available backend. + // If the policy name is set via the client API, that value overrides + // the value specified here. + // + // Note that if the resolver returns at least one balancer address (as + // opposed to backend addresses), gRPC will use grpclb (see + // https://github.com/grpc/grpc/blob/master/doc/load-balancing.md), + // regardless of what LB policy is requested either here or via the + // client API. However, if the resolver returns backend addresses as + // well as balancer addresses, the client may fall back to the requested + // policy if it is unable to reach any of the grpclb load balancers. 'loadBalancingPolicy': string, // Per-method configuration. Optional. diff --git a/src/core/ext/client_channel/client_channel.c b/src/core/ext/client_channel/client_channel.c index 00c20913b0f..7df211d0000 100644 --- a/src/core/ext/client_channel/client_channel.c +++ b/src/core/ext/client_channel/client_channel.c @@ -368,27 +368,25 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, GPR_ASSERT(channel_arg->type == GRPC_ARG_STRING); lb_policy_name = channel_arg->value.string; } - // Special case: If all of the addresses are balancer addresses, - // assume that we should use the grpclb policy, regardless of what the - // resolver actually specified. + // Special case: If at least one balancer address is present, we use + // the grpclb policy, regardless of what the resolver actually specified. channel_arg = grpc_channel_args_find(chand->resolver_result, GRPC_ARG_LB_ADDRESSES); if (channel_arg != NULL) { GPR_ASSERT(channel_arg->type == GRPC_ARG_POINTER); grpc_lb_addresses *addresses = channel_arg->value.pointer.p; - bool found_backend_address = false; + bool found_balancer_address = false; for (size_t i = 0; i < addresses->num_addresses; ++i) { - if (!addresses->addresses[i].is_balancer) { - found_backend_address = true; + if (addresses->addresses[i].is_balancer) { + found_balancer_address = true; break; } } - if (!found_backend_address) { + if (found_balancer_address) { if (lb_policy_name != NULL && strcmp(lb_policy_name, "grpclb") != 0) { gpr_log(GPR_INFO, - "resolver requested LB policy %s but provided only balancer " - "addresses, no backend addresses -- forcing use of grpclb LB " - "policy", + "resolver requested LB policy %s but provided at least one " + "balancer address -- forcing use of grpclb LB policy", lb_policy_name); } lb_policy_name = "grpclb"; diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index d612591f2eb..e0e5e28eb6f 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -841,10 +841,10 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, /* Count the number of gRPC-LB addresses. There must be at least one. * TODO(roth): For now, we ignore non-balancer addresses, but in the * future, we may change the behavior such that we fall back to using - * the non-balancer addresses if we cannot reach any balancers. At that - * time, this should be changed to allow a list with no balancer addresses, - * since the resolver might fail to return a balancer address even when - * this is the right LB policy to use. */ + * the non-balancer addresses if we cannot reach any balancers. In the + * fallback case, we should use the LB policy indicated by + * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is + * unset, we should default to pick_first). */ const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); GPR_ASSERT(arg != NULL && arg->type == GRPC_ARG_POINTER); From e3ec4b2c287b480d61ddaa2b461cf6d468519313 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Fri, 24 Mar 2017 07:33:38 -0700 Subject: [PATCH 03/51] Clarify wording. --- doc/service_config.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/service_config.md b/doc/service_config.md index c5d4ef42b19..8039fcad095 100644 --- a/doc/service_config.md +++ b/doc/service_config.md @@ -24,9 +24,10 @@ The service config is a JSON string of the following form: // opposed to backend addresses), gRPC will use grpclb (see // https://github.com/grpc/grpc/blob/master/doc/load-balancing.md), // regardless of what LB policy is requested either here or via the - // client API. However, if the resolver returns backend addresses as - // well as balancer addresses, the client may fall back to the requested - // policy if it is unable to reach any of the grpclb load balancers. + // client API. However, if the resolver returns at least one backend + // address in addition to the balancer address(es), the client may fall + // back to the requested policy if it is unable to reach any of the + // grpclb load balancers. 'loadBalancingPolicy': string, // Per-method configuration. Optional. From 895f3d83da12664e59379d9ce6cccb8ab9f244b7 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 5 Apr 2017 13:12:30 -0700 Subject: [PATCH 04/51] Support configuring dispatch queue in GRPCCall and below --- src/objective-c/GRPCClient/GRPCCall.h | 7 +++++++ src/objective-c/GRPCClient/GRPCCall.m | 16 +++++++++++++++- .../RxLibrary/GRXConcurrentWriteable.h | 4 +++- .../RxLibrary/GRXConcurrentWriteable.m | 10 ++++++++-- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 7645bb1d34a..85c5b47c8ea 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -253,6 +253,13 @@ extern id const kGRPCTrailersKey; */ + (void)setCallSafety:(GRPCCallSafety)callSafety host:(NSString *)host path:(NSString *)path; +/** + * Set the dispatch queue to be used for queue responses. + * + * This configuration is only effective before the call starts. + */ +- (void)setResponseDispatchQueue:(dispatch_queue_t)queue; + // TODO(jcanizales): Let specify a deadline. As a category of GRXWriter? @end diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 051138ea4da..f9d13fea578 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -113,6 +113,10 @@ static NSMutableDictionary *callFlags; // the SendClose op is added. BOOL _unaryCall; NSMutableArray *_unaryOpBatch; + + // The dispatch queue to be used for enqueuing responses to user. Defaulted to the main dispatch + // queue + dispatch_queue_t _responseQueue; } @synthesize state = _state; @@ -175,10 +179,19 @@ static NSMutableDictionary *callFlags; _unaryCall = YES; _unaryOpBatch = [NSMutableArray arrayWithCapacity:kMaxClientBatch]; } + + _responseQueue = dispatch_get_main_queue(); } return self; } +- (void)setResponseDispatchQueue:(dispatch_queue_t)queue { + if (_state != GRXWriterStateNotStarted) { + return; + } + _responseQueue = queue; +} + #pragma mark Finish - (void)finishWithError:(NSError *)errorOrNil { @@ -424,7 +437,8 @@ static NSMutableDictionary *callFlags; // that the life of the instance is determined by this retain cycle. _retainSelf = self; - _responseWriteable = [[GRXConcurrentWriteable alloc] initWithWriteable:writeable]; + _responseWriteable = [[GRXConcurrentWriteable alloc] initWithWriteable:writeable + dispatchQueue:_responseQueue]; _wrappedCall = [[GRPCWrappedCall alloc] initWithHost:_host path:_path]; NSAssert(_wrappedCall, @"Error allocating RPC objects. Low memory?"); diff --git a/src/objective-c/RxLibrary/GRXConcurrentWriteable.h b/src/objective-c/RxLibrary/GRXConcurrentWriteable.h index b2775f98b56..07004f6d4dc 100644 --- a/src/objective-c/RxLibrary/GRXConcurrentWriteable.h +++ b/src/objective-c/RxLibrary/GRXConcurrentWriteable.h @@ -53,7 +53,9 @@ * The GRXWriteable instance is retained until writesFinishedWithError: is sent to it, and released * after that. */ -- (instancetype)initWithWriteable:(id)writeable NS_DESIGNATED_INITIALIZER; +- (instancetype)initWithWriteable:(id)writeable + dispatchQueue:(dispatch_queue_t)queue NS_DESIGNATED_INITIALIZER; +- (instancetype)initWithWriteable:(id)writeable; /** * Enqueues writeValue: to be sent to the writeable in the main thread. diff --git a/src/objective-c/RxLibrary/GRXConcurrentWriteable.m b/src/objective-c/RxLibrary/GRXConcurrentWriteable.m index 08bd079aea5..88aa7a7282f 100644 --- a/src/objective-c/RxLibrary/GRXConcurrentWriteable.m +++ b/src/objective-c/RxLibrary/GRXConcurrentWriteable.m @@ -51,14 +51,20 @@ } // Designated initializer -- (instancetype)initWithWriteable:(id)writeable { +- (instancetype)initWithWriteable:(id)writeable + dispatchQueue:(dispatch_queue_t)queue { if (self = [super init]) { - _writeableQueue = dispatch_get_main_queue(); + _writeableQueue = queue; _writeable = writeable; } return self; } +- (instancetype)initWithWriteable:(id)writeable { + return [self initWithWriteable:writeable + dispatchQueue:dispatch_get_main_queue()]; +} + - (void)enqueueValue:(id)value completionHandler:(void (^)())handler { dispatch_async(_writeableQueue, ^{ // We're racing a possible cancellation performed by another thread. To turn all already- From 56369ea4a2c785747d656943f469b461085c3d8d Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 6 Apr 2017 11:52:43 -0700 Subject: [PATCH 05/51] Support in ProtoRPC --- src/compiler/objective_c_generator.cc | 27 +++++++++++++++++++-------- src/compiler/objective_c_plugin.cc | 1 + 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/compiler/objective_c_generator.cc b/src/compiler/objective_c_generator.cc index 1d7faf120dc..03a0762a38b 100644 --- a/src/compiler/objective_c_generator.cc +++ b/src/compiler/objective_c_generator.cc @@ -149,17 +149,19 @@ void PrintMethodDeclarations(Printer *printer, const MethodDescriptor *method) { void PrintSimpleImplementation(Printer *printer, const MethodDescriptor *method, map< ::grpc::string, ::grpc::string> vars) { printer->Print("{\n"); - printer->Print(vars, " [[self RPCTo$method_name$With"); + printer->Print(vars, " GRPCProtoCall *rpc = [self RPCTo$method_name$With"); if (method->client_streaming()) { printer->Print("RequestsWriter:requestWriter"); } else { printer->Print("Request:request"); } if (method->server_streaming()) { - printer->Print(" eventHandler:eventHandler] start];\n"); + printer->Print(" eventHandler:eventHandler];\n"); } else { - printer->Print(" handler:handler] start];\n"); + printer->Print(" handler:handler];\n"); } + printer->Print(" [rpc setResponseDispatchQueue:_defaultResponseDispatchQueue];\n"); + printer->Print(" [rpc start];\n"); printer->Print("}\n"); } @@ -167,23 +169,25 @@ void PrintAdvancedImplementation(Printer *printer, const MethodDescriptor *method, map< ::grpc::string, ::grpc::string> vars) { printer->Print("{\n"); - printer->Print(vars, " return [self RPCToMethod:@\"$method_name$\"\n"); + printer->Print(vars, " GRPCProtoCall *rpc = [self RPCToMethod:@\"$method_name$\"\n"); - printer->Print(" requestsWriter:"); + printer->Print(" requestsWriter:"); if (method->client_streaming()) { printer->Print("requestWriter\n"); } else { printer->Print("[GRXWriter writerWithValue:request]\n"); } - printer->Print(vars, " responseClass:[$response_class$ class]\n"); + printer->Print(vars, " responseClass:[$response_class$ class]\n"); - printer->Print(" responsesWriteable:[GRXWriteable "); + printer->Print(" responsesWriteable:[GRXWriteable "); if (method->server_streaming()) { printer->Print("writeableWithEventHandler:eventHandler]];\n"); } else { printer->Print("writeableWithSingleHandler:handler]];\n"); } + printer->Print(" [rpc setResponseDispatchQueue:_defaultResponseDispatchQueue];\n"); + printer->Print(" return rpc;\n"); printer->Print("}\n"); } @@ -234,6 +238,7 @@ void PrintMethodImplementations(Printer *printer, "- (instancetype)initWithHost:(NSString *)host" " NS_DESIGNATED_INITIALIZER;\n"); printer.Print("+ (instancetype)serviceWithHost:(NSString *)host;\n"); + printer.Print("- (void)setDefaultResponseDispatchQueue:(dispatch_queue_t)queue;\n"); printer.Print("@end\n"); } return output; @@ -251,12 +256,15 @@ void PrintMethodImplementations(Printer *printer, {"service_class", ServiceClassName(service)}, {"package", service->file()->package()}}; - printer.Print(vars, "@implementation $service_class$\n\n"); + printer.Print(vars, "@implementation $service_class$ {\n"); + printer.Print(vars, " dispatch_queue_t _defaultResponseDispatchQueue;\n"); + printer.Print(vars, "}\n\n"); printer.Print("// Designated initializer\n"); printer.Print("- (instancetype)initWithHost:(NSString *)host {\n"); printer.Print( vars, + " _defaultResponseDispatchQueue = dispatch_get_main_queue();\n" " return (self = [super initWithHost:host" " packageName:@\"$package$\" serviceName:@\"$service_name$\"]);\n"); printer.Print("}\n\n"); @@ -270,6 +278,9 @@ void PrintMethodImplementations(Printer *printer, printer.Print("}\n\n"); printer.Print("+ (instancetype)serviceWithHost:(NSString *)host {\n"); printer.Print(" return [[self alloc] initWithHost:host];\n"); + printer.Print("}\n\n"); + printer.Print("- (void)setDefaultResponseDispatchQueue:(dispatch_queue_t)queue {\n"); + printer.Print(" _defaultResponseDispatchQueue = queue;\n"); printer.Print("}\n\n\n"); for (int i = 0; i < service->method_count(); i++) { diff --git a/src/compiler/objective_c_plugin.cc b/src/compiler/objective_c_plugin.cc index 8de0997ebea..5178115e44c 100644 --- a/src/compiler/objective_c_plugin.cc +++ b/src/compiler/objective_c_plugin.cc @@ -68,6 +68,7 @@ class ObjectiveCGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { ::grpc::string imports = ::grpc::string("#import \"") + file_name + ".pbobjc.h\"\n\n" "#import \n" + "#import \n" "#import \n" "#import \n"; From e1ed7712102363d5e6fdad31b1016119f95f0631 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 6 Apr 2017 11:52:55 -0700 Subject: [PATCH 06/51] Add test --- src/objective-c/tests/InteropTests.m | 95 ++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index 91053568690..4466d6810cc 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -457,4 +457,99 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +- (void)testAlternateDispatchQueue { + XCTAssertNotNil(self.class.host); + __weak XCTestExpectation *expectation1 = [self expectationWithDescription:@"AlternateDispatchQueue1"]; + + NSArray *requests = @[@27182, @8, @1828, @45904]; + NSArray *responses = @[@31415, @9, @2653, @58979]; + + // Set the default dispatch queue + NSString *queue1_label = @"test.queue1"; + NSString *queue2_label = @"test.queue2"; + dispatch_queue_t queue1 = dispatch_queue_create([queue1_label UTF8String], DISPATCH_QUEUE_SERIAL); + dispatch_queue_t queue2 = dispatch_queue_create([queue2_label UTF8String], DISPATCH_QUEUE_SERIAL); + [_service setDefaultResponseDispatchQueue:queue1]; + GRXBufferedPipe *requestsBuffer1 = [[GRXBufferedPipe alloc] init]; + + __block int index = 0; + + id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:requests[index] + requestedResponseSize:responses[index]]; + [requestsBuffer1 writeValue:request]; + + [_service fullDuplexCallWithRequestsWriter:requestsBuffer1 + eventHandler:^(BOOL done, + RMTStreamingOutputCallResponse *response, + NSError *error) { + XCTAssertNil(error, @"Finished with unexpected error: %@", error); + XCTAssertTrue(done || response, @"Event handler called without an event."); + NSString *label = [NSString stringWithUTF8String:dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL)]; + XCTAssert([label isEqualToString:queue1_label]); + + if (response) { + XCTAssertLessThan(index, 4, @"More than 4 responses received."); + id expected = [RMTStreamingOutputCallResponse messageWithPayloadSize:responses[index]]; + XCTAssertEqualObjects(response, expected); + index += 1; + if (index < 4) { + id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:requests[index] + requestedResponseSize:responses[index]]; + [requestsBuffer1 writeValue:request]; + } else { + [requestsBuffer1 writesFinishedWithError:nil]; + } + } + + if (done) { + XCTAssertEqual(index, 4, @"Received %i responses instead of 4.", index); + [expectation1 fulfill]; + } + }]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; + + // Test overriding default queue with another queue + __weak XCTestExpectation *expectation2 = [self expectationWithDescription:@"AlternateDispatchQueue2"]; + GRXBufferedPipe *requestsBuffer2 = [[GRXBufferedPipe alloc] init]; + + index = 0; + + [requestsBuffer2 writeValue:request]; + + GRPCProtoCall *rpc = [_service RPCToFullDuplexCallWithRequestsWriter:requestsBuffer2 + eventHandler:^(BOOL done, + RMTStreamingOutputCallResponse *response, + NSError *error) { + XCTAssertNil(error, @"Finished with unexpected error: %@", error); + XCTAssertTrue(done || response, @"Event handler called without an event."); + NSString *label = [NSString stringWithUTF8String:dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL)]; + XCTAssert([label isEqualToString:queue2_label]); + + if (response) { + XCTAssertLessThan(index, 4, @"More than 4 responses received."); + id expected = [RMTStreamingOutputCallResponse messageWithPayloadSize:responses[index]]; + XCTAssertEqualObjects(response, expected); + index += 1; + if (index < 4) { + id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:requests[index] + requestedResponseSize:responses[index]]; + [requestsBuffer2 writeValue:request]; + } else { + [requestsBuffer2 writesFinishedWithError:nil]; + } + } + + if (done) { + XCTAssertEqual(index, 4, @"Received %i responses instead of 4.", index); + [expectation2 fulfill]; + } + }]; + [rpc setResponseDispatchQueue:queue2]; + [rpc start]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; + [_service setDefaultResponseDispatchQueue:dispatch_get_main_queue()]; +} + @end From 2f23be7e8c9a1342b759e5fbb73963a51ce1eddb Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Sun, 9 Apr 2017 15:57:31 -0700 Subject: [PATCH 07/51] clang-format --- src/compiler/objective_c_generator.cc | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/compiler/objective_c_generator.cc b/src/compiler/objective_c_generator.cc index 03a0762a38b..ba89b28b7b5 100644 --- a/src/compiler/objective_c_generator.cc +++ b/src/compiler/objective_c_generator.cc @@ -160,7 +160,8 @@ void PrintSimpleImplementation(Printer *printer, const MethodDescriptor *method, } else { printer->Print(" handler:handler];\n"); } - printer->Print(" [rpc setResponseDispatchQueue:_defaultResponseDispatchQueue];\n"); + printer->Print( + " [rpc setResponseDispatchQueue:_defaultResponseDispatchQueue];\n"); printer->Print(" [rpc start];\n"); printer->Print("}\n"); } @@ -169,7 +170,8 @@ void PrintAdvancedImplementation(Printer *printer, const MethodDescriptor *method, map< ::grpc::string, ::grpc::string> vars) { printer->Print("{\n"); - printer->Print(vars, " GRPCProtoCall *rpc = [self RPCToMethod:@\"$method_name$\"\n"); + printer->Print( + vars, " GRPCProtoCall *rpc = [self RPCToMethod:@\"$method_name$\"\n"); printer->Print(" requestsWriter:"); if (method->client_streaming()) { @@ -178,7 +180,9 @@ void PrintAdvancedImplementation(Printer *printer, printer->Print("[GRXWriter writerWithValue:request]\n"); } - printer->Print(vars, " responseClass:[$response_class$ class]\n"); + printer->Print( + vars, + " responseClass:[$response_class$ class]\n"); printer->Print(" responsesWriteable:[GRXWriteable "); if (method->server_streaming()) { @@ -186,7 +190,8 @@ void PrintAdvancedImplementation(Printer *printer, } else { printer->Print("writeableWithSingleHandler:handler]];\n"); } - printer->Print(" [rpc setResponseDispatchQueue:_defaultResponseDispatchQueue];\n"); + printer->Print( + " [rpc setResponseDispatchQueue:_defaultResponseDispatchQueue];\n"); printer->Print(" return rpc;\n"); printer->Print("}\n"); @@ -238,7 +243,8 @@ void PrintMethodImplementations(Printer *printer, "- (instancetype)initWithHost:(NSString *)host" " NS_DESIGNATED_INITIALIZER;\n"); printer.Print("+ (instancetype)serviceWithHost:(NSString *)host;\n"); - printer.Print("- (void)setDefaultResponseDispatchQueue:(dispatch_queue_t)queue;\n"); + printer.Print( + "- (void)setDefaultResponseDispatchQueue:(dispatch_queue_t)queue;\n"); printer.Print("@end\n"); } return output; @@ -279,7 +285,8 @@ void PrintMethodImplementations(Printer *printer, printer.Print("+ (instancetype)serviceWithHost:(NSString *)host {\n"); printer.Print(" return [[self alloc] initWithHost:host];\n"); printer.Print("}\n\n"); - printer.Print("- (void)setDefaultResponseDispatchQueue:(dispatch_queue_t)queue {\n"); + printer.Print( + "- (void)setDefaultResponseDispatchQueue:(dispatch_queue_t)queue {\n"); printer.Print(" _defaultResponseDispatchQueue = queue;\n"); printer.Print("}\n\n\n"); From 2f40ff423ece50964d1041a4ad68939260da9fe6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 16:01:19 -0700 Subject: [PATCH 08/51] Support making hybrid cqs in core --- src/core/lib/surface/completion_queue.c | 167 +++++++++++++++++++++--- src/core/lib/surface/completion_queue.h | 1 - 2 files changed, 149 insertions(+), 19 deletions(-) diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index 35e9f7eb308..e17f094837e 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -60,13 +60,145 @@ typedef struct { void *tag; } plucker; +typedef struct { + size_t (*size)(void); + void (*init)(grpc_pollset *pollset, gpr_mu **mu); + grpc_error *(*kick)(grpc_pollset *pollset, + grpc_pollset_worker *specific_worker); + grpc_error *(*work)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_pollset_worker **worker, gpr_timespec now, + gpr_timespec deadline); + void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, + grpc_closure *closure); + void (*destroy)(grpc_pollset *pollset); +} cq_poller_vtable; + +typedef struct non_polling_worker { + gpr_cv cv; + bool kicked; + struct non_polling_worker *next; + struct non_polling_worker *prev; +} non_polling_worker; + +typedef struct { + gpr_mu mu; + non_polling_worker *root; + grpc_closure *shutdown; +} non_polling_poller; + +static size_t non_polling_poller_size(void) { + return sizeof(non_polling_poller); +} + +static void non_polling_poller_init(grpc_pollset *pollset, gpr_mu **mu) { + non_polling_poller *npp = (non_polling_poller *)pollset; + gpr_mu_init(&npp->mu); + *mu = &npp->mu; +} + +static void non_polling_poller_destroy(grpc_pollset *pollset) { + non_polling_poller *npp = (non_polling_poller *)pollset; + gpr_mu_destroy(&npp->mu); +} + +static grpc_error *non_polling_poller_work(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_pollset_worker **worker, + gpr_timespec now, + gpr_timespec deadline) { + non_polling_poller *npp = (non_polling_poller *)pollset; + non_polling_worker w; + gpr_cv_init(&w.cv); + *worker = (grpc_pollset_worker *)&w; + if (npp->root == NULL) { + npp->root = w.next = w.prev = &w; + } else { + w.next = npp->root; + w.prev = w.next->prev; + w.next->prev = w.prev->next = &w; + } + w.kicked = false; + while (!npp->shutdown && !w.kicked && !gpr_cv_wait(&w.cv, &npp->mu, deadline)) + ; + if (&w == npp->root) { + npp->root = w.next; + if (&w == npp->root) { + if (npp->shutdown) { + grpc_closure_sched(exec_ctx, npp->shutdown, GRPC_ERROR_NONE); + } + npp->root = NULL; + } + w.next->prev = w.prev; + w.prev->next = w.next; + } + gpr_cv_destroy(&w.cv); + *worker = NULL; + return GRPC_ERROR_NONE; +} + +static grpc_error *non_polling_poller_kick( + grpc_pollset *pollset, grpc_pollset_worker *specific_worker) { + non_polling_poller *p = (non_polling_poller *)pollset; + if (specific_worker == NULL) specific_worker = (grpc_pollset_worker *)p->root; + if (specific_worker != NULL) { + non_polling_worker *w = (non_polling_worker *)specific_worker; + if (!w->kicked) { + w->kicked = true; + gpr_cv_signal(&w->cv); + } + } + return GRPC_ERROR_NONE; +} + +static void non_polling_poller_shutdown(grpc_exec_ctx *exec_ctx, + grpc_pollset *pollset, + grpc_closure *closure) { + non_polling_poller *p = (non_polling_poller *)pollset; + GPR_ASSERT(closure != NULL); + p->shutdown = closure; + if (p->root == NULL) { + grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); + } else { + non_polling_worker *w = p->root; + do { + gpr_cv_signal(&w->cv); + w = w->next; + } while (w != p->root); + } +} + +static const cq_poller_vtable g_poller_vtable_by_poller_type[] = { + /* GRPC_CQ_DEFAULT_POLLING */ + {.size = grpc_pollset_size, + .init = grpc_pollset_init, + .kick = grpc_pollset_kick, + .work = grpc_pollset_work, + .shutdown = grpc_pollset_shutdown, + .destroy = grpc_pollset_destroy}, + /* GRPC_CQ_NON_LISTENING */ + {.size = grpc_pollset_size, + .init = grpc_pollset_init, + .kick = grpc_pollset_kick, + .work = grpc_pollset_work, + .shutdown = grpc_pollset_shutdown, + .destroy = grpc_pollset_destroy}, + /* GRPC_CQ_NON_POLLING */ + {.size = non_polling_poller_size, + .init = non_polling_poller_init, + .kick = non_polling_poller_kick, + .work = non_polling_poller_work, + .shutdown = non_polling_poller_shutdown, + .destroy = non_polling_poller_destroy}, +}; + /* Completion queue structure */ struct grpc_completion_queue { /** owned by pollset */ gpr_mu *mu; grpc_cq_completion_type completion_type; - grpc_cq_polling_type polling_type; + + const cq_poller_vtable *poller_vtable; /** completed events */ grpc_cq_completion completed_head; @@ -127,15 +259,18 @@ grpc_completion_queue *grpc_completion_queue_create_internal( "polling_type=%d)", 2, (completion_type, polling_type)); - cc = gpr_zalloc(sizeof(grpc_completion_queue) + grpc_pollset_size()); - grpc_pollset_init(POLLSET_FROM_CQ(cc), &cc->mu); + const cq_poller_vtable *poller_vtable = + &g_poller_vtable_by_poller_type[polling_type]; + + cc = gpr_zalloc(sizeof(grpc_completion_queue) + poller_vtable->size()); + poller_vtable->init(POLLSET_FROM_CQ(cc), &cc->mu); #ifndef NDEBUG cc->outstanding_tags = NULL; cc->outstanding_tag_capacity = 0; #endif cc->completion_type = completion_type; - cc->polling_type = polling_type; + cc->poller_vtable = poller_vtable; /* Initial ref is dropped by grpc_completion_queue_shutdown */ gpr_ref_init(&cc->pending_events, 1); @@ -164,10 +299,6 @@ grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue *cc) { return cc->completion_type; } -grpc_cq_polling_type grpc_get_cq_polling_type(grpc_completion_queue *cc) { - return cc->polling_type; -} - #ifdef GRPC_CQ_REF_COUNT_DEBUG void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason, const char *file, int line) { @@ -195,7 +326,7 @@ void grpc_cq_internal_unref(grpc_completion_queue *cc) { #endif if (gpr_unref(&cc->owning_refs)) { GPR_ASSERT(cc->completed_head.next == (uintptr_t)&cc->completed_head); - grpc_pollset_destroy(POLLSET_FROM_CQ(cc)); + cc->poller_vtable->destroy(POLLSET_FROM_CQ(cc)); #ifndef NDEBUG gpr_free(cc->outstanding_tags); #endif @@ -280,7 +411,7 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, } } grpc_error *kick_error = - grpc_pollset_kick(POLLSET_FROM_CQ(cc), pluck_worker); + cc->poller_vtable->kick(POLLSET_FROM_CQ(cc), pluck_worker); gpr_mu_unlock(cc->mu); if (kick_error != GRPC_ERROR_NONE) { const char *msg = grpc_error_string(kick_error); @@ -295,8 +426,8 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, GPR_ASSERT(!cc->shutdown); GPR_ASSERT(cc->shutdown_called); cc->shutdown = 1; - grpc_pollset_shutdown(exec_ctx, POLLSET_FROM_CQ(cc), - &cc->pollset_shutdown_done); + cc->poller_vtable->shutdown(exec_ctx, POLLSET_FROM_CQ(cc), + &cc->pollset_shutdown_done); gpr_mu_unlock(cc->mu); } @@ -452,8 +583,8 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_mu_lock(cc->mu); continue; } else { - grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), NULL, - now, iteration_deadline); + grpc_error *err = cc->poller_vtable->work(&exec_ctx, POLLSET_FROM_CQ(cc), + NULL, now, iteration_deadline); if (err != GRPC_ERROR_NONE) { gpr_mu_unlock(cc->mu); const char *msg = grpc_error_string(err); @@ -644,8 +775,8 @@ grpc_event grpc_completion_queue_pluck(grpc_completion_queue *cc, void *tag, grpc_exec_ctx_flush(&exec_ctx); gpr_mu_lock(cc->mu); } else { - grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), - &worker, now, iteration_deadline); + grpc_error *err = cc->poller_vtable->work( + &exec_ctx, POLLSET_FROM_CQ(cc), &worker, now, iteration_deadline); if (err != GRPC_ERROR_NONE) { del_plucker(cc, tag, &worker); gpr_mu_unlock(cc->mu); @@ -689,8 +820,8 @@ void grpc_completion_queue_shutdown(grpc_completion_queue *cc) { if (gpr_unref(&cc->pending_events)) { GPR_ASSERT(!cc->shutdown); cc->shutdown = 1; - grpc_pollset_shutdown(&exec_ctx, POLLSET_FROM_CQ(cc), - &cc->pollset_shutdown_done); + cc->poller_vtable->shutdown(&exec_ctx, POLLSET_FROM_CQ(cc), + &cc->pollset_shutdown_done); } gpr_mu_unlock(cc->mu); grpc_exec_ctx_finish(&exec_ctx); diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index 1ff3d64293a..0995a56889b 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -100,7 +100,6 @@ void grpc_cq_mark_server_cq(grpc_completion_queue *cc); int grpc_cq_is_server_cq(grpc_completion_queue *cc); grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue *cc); -grpc_cq_polling_type grpc_get_cq_polling_type(grpc_completion_queue *cc); grpc_completion_queue *grpc_completion_queue_create_internal( grpc_cq_completion_type completion_type, grpc_cq_polling_type polling_type); From 334c4678a326284c17254e91a4498728a9c67f69 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 16:26:07 -0700 Subject: [PATCH 09/51] Start building out C++ interface --- .../grpc++/impl/codegen/completion_queue.h | 12 ++++++--- include/grpc/grpc.h | 26 ------------------- include/grpc/impl/codegen/grpc_types.h | 26 +++++++++++++++++++ src/cpp/server/server_builder.cc | 8 ++++-- 4 files changed, 40 insertions(+), 32 deletions(-) diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h index 61617f2bdc9..906a64693e1 100644 --- a/include/grpc++/impl/codegen/completion_queue.h +++ b/include/grpc++/impl/codegen/completion_queue.h @@ -289,17 +289,21 @@ class CompletionQueue : private GrpcLibraryCodegen { /// by servers. Instantiated by \a ServerBuilder. class ServerCompletionQueue : public CompletionQueue { public: - bool IsFrequentlyPolled() { return is_frequently_polled_; } + bool IsFrequentlyPolled() { return polling_type_ != GRPC_CQ_NON_LISTENING; } private: - bool is_frequently_polled_; + grpc_cq_polling_type polling_type_; friend class ServerBuilder; /// \param is_frequently_polled Informs the GRPC library about whether the /// server completion queue would be actively polled (by calling Next() or /// AsyncNext()). By default all server completion queues are assumed to be /// frequently polled. - ServerCompletionQueue(bool is_frequently_polled = true) - : is_frequently_polled_(is_frequently_polled) {} + ServerCompletionQueue(grpc_cq_polling_type polling_type) + : CompletionQueue(MakeCompletionQueue(polling_type)), + polling_type_(polling_type) {} + + static grpc_completion_queue* MakeCompletionQueue( + grpc_cq_polling_type polling_type); }; } // namespace grpc diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 42cf201da44..f3201edad29 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -102,32 +102,6 @@ typedef enum { GRPC_CQ_PLUCK } grpc_cq_completion_type; -/** Completion queues internally MAY maintain a set of file descriptors in a - structure called 'pollset'. This enum specifies if a completion queue has an - associated pollset and any restrictions on the type of file descriptors that - can be present in the pollset. - - I/O progress can only be made when grpc_completion_queue_next() or - grpc_completion_queue_pluck() are called on the completion queue (unless the - grpc_cq_polling_type is GRPC_CQ_NON_POLLING) and hence it is very important - to actively call these APIs */ -typedef enum { - /** The completion queue will have an associated pollset and there is no - restriction on the type of file descriptors the pollset may contain */ - GRPC_CQ_DEFAULT_POLLING, - - /** Similar to GRPC_CQ_DEFAULT_POLLING except that the completion queues will - not contain any 'listening file descriptors' (i.e file descriptors used to - listen to incoming channels) */ - GRPC_CQ_NON_LISTENING, - - /** The completion queue will not have an associated pollset. Note that - grpc_completion_queue_next() or grpc_completion_queue_pluck() MUST still - be called to pop events from the completion queue; it is not required to - call them actively to make I/O progress */ - GRPC_CQ_NON_POLLING -} grpc_cq_polling_type; - #define GRPC_CQ_CURRENT_VERSION 1 typedef struct grpc_completion_queue_attributes { /* The version number of this structure. More fields might be added to this diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 4383691a837..02732205f5f 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -543,6 +543,32 @@ typedef struct { typedef struct grpc_resource_quota grpc_resource_quota; +/** Completion queues internally MAY maintain a set of file descriptors in a + structure called 'pollset'. This enum specifies if a completion queue has an + associated pollset and any restrictions on the type of file descriptors that + can be present in the pollset. + + I/O progress can only be made when grpc_completion_queue_next() or + grpc_completion_queue_pluck() are called on the completion queue (unless the + grpc_cq_polling_type is GRPC_CQ_NON_POLLING) and hence it is very important + to actively call these APIs */ +typedef enum { + /** The completion queue will have an associated pollset and there is no + restriction on the type of file descriptors the pollset may contain */ + GRPC_CQ_DEFAULT_POLLING, + + /** Similar to GRPC_CQ_DEFAULT_POLLING except that the completion queues will + not contain any 'listening file descriptors' (i.e file descriptors used to + listen to incoming channels) */ + GRPC_CQ_NON_LISTENING, + + /** The completion queue will not have an associated pollset. Note that + grpc_completion_queue_next() or grpc_completion_queue_pluck() MUST still + be called to pop events from the completion queue; it is not required to + call them actively to make I/O progress */ + GRPC_CQ_NON_POLLING +} grpc_cq_polling_type; + #ifdef __cplusplus } #endif diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc index 4eb4b5a1b21..c6784ea1593 100644 --- a/src/cpp/server/server_builder.cc +++ b/src/cpp/server/server_builder.cc @@ -83,7 +83,8 @@ ServerBuilder::~ServerBuilder() { std::unique_ptr ServerBuilder::AddCompletionQueue( bool is_frequently_polled) { - ServerCompletionQueue* cq = new ServerCompletionQueue(is_frequently_polled); + ServerCompletionQueue* cq = new ServerCompletionQueue( + is_frequently_polled ? GRPC_CQ_DEFAULT_POLLING : GRPC_CQ_NON_LISTENING); cqs_.push_back(cq); return std::unique_ptr(cq); } @@ -251,9 +252,12 @@ std::unique_ptr ServerBuilder::BuildAndStart() { sync_server_settings_.max_pollers, sync_server_settings_.cq_timeout_msec); + grpc_cq_polling_type polling_type = + cqs_.empty() ? GRPC_CQ_DEFAULT_POLLING : GRPC_CQ_NON_POLLING; + // Create completion queues to listen to incoming rpc requests for (int i = 0; i < sync_server_settings_.num_cqs; i++) { - sync_server_cqs->emplace_back(new ServerCompletionQueue()); + sync_server_cqs->emplace_back(new ServerCompletionQueue(polling_type)); } } From 75bfb9754827ef63ede77e27ad901e3355536419 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 17:55:12 -0700 Subject: [PATCH 10/51] Finish hybrid server stuff, ensure it gets tested --- .../grpc++/impl/codegen/client_unary_call.h | 4 ++- .../grpc++/impl/codegen/completion_queue.h | 32 +++++++++---------- include/grpc++/impl/codegen/core_codegen.h | 9 +++++- .../impl/codegen/core_codegen_interface.h | 6 ++++ include/grpc++/impl/codegen/sync_stream.h | 12 +++++-- include/grpc/grpc.h | 23 ------------- include/grpc/impl/codegen/grpc_types.h | 23 +++++++++++++ src/core/lib/surface/completion_queue.c | 21 +++++++----- src/core/lib/surface/server.c | 7 ++-- src/cpp/common/core_codegen.cc | 12 +++++++ src/cpp/server/server_builder.cc | 26 +++++++++++---- test/cpp/end2end/async_end2end_test.cc | 28 ++++++++++++---- 12 files changed, 134 insertions(+), 69 deletions(-) diff --git a/include/grpc++/impl/codegen/client_unary_call.h b/include/grpc++/impl/codegen/client_unary_call.h index a5a4f3d7398..4bf35ae7785 100644 --- a/include/grpc++/impl/codegen/client_unary_call.h +++ b/include/grpc++/impl/codegen/client_unary_call.h @@ -52,7 +52,9 @@ template Status BlockingUnaryCall(ChannelInterface* channel, const RpcMethod& method, ClientContext* context, const InputMessage& request, OutputMessage* result) { - CompletionQueue cq(true); // Pluckable completion queue + CompletionQueue cq(grpc_completion_queue_attributes{ + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, + GRPC_CQ_DEFAULT_POLLING}); // Pluckable completion queue Call call(channel->CreateCall(method, context, &cq)); CallOpSet, diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h index 906a64693e1..c8ab726b0f4 100644 --- a/include/grpc++/impl/codegen/completion_queue.h +++ b/include/grpc++/impl/codegen/completion_queue.h @@ -102,7 +102,9 @@ class CompletionQueue : private GrpcLibraryCodegen { public: /// Default constructor. Implicitly creates a \a grpc_completion_queue /// instance. - CompletionQueue() : CompletionQueue(false) {} + CompletionQueue() + : CompletionQueue(grpc_completion_queue_attributes{ + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, GRPC_CQ_DEFAULT_POLLING}) {} /// Wrap \a take, taking ownership of the instance. /// @@ -182,6 +184,16 @@ class CompletionQueue : private GrpcLibraryCodegen { }; void CompleteAvalanching(); + protected: + /// Private constructor of CompletionQueue only visible to friend classes + CompletionQueue(const grpc_completion_queue_attributes& attributes) { + cq_ = g_core_codegen_interface->grpc_completion_queue_create( + g_core_codegen_interface->grpc_completion_queue_factory_lookup( + &attributes), + &attributes, NULL); + InitialAvalanching(); // reserve this for the future shutdown + } + private: // Friend synchronous wrappers so that they can access Pluck(), which is // a semi-private API geared towards the synchronous implementation. @@ -215,18 +227,6 @@ class CompletionQueue : private GrpcLibraryCodegen { const InputMessage& request, OutputMessage* result); - /// Private constructor of CompletionQueue only visible to friend classes - CompletionQueue(bool is_pluck) { - if (is_pluck) { - cq_ = g_core_codegen_interface->grpc_completion_queue_create_for_pluck( - nullptr); - } else { - cq_ = g_core_codegen_interface->grpc_completion_queue_create_for_next( - nullptr); - } - InitialAvalanching(); // reserve this for the future shutdown - } - NextStatus AsyncNextInternal(void** tag, bool* ok, gpr_timespec deadline); /// Wraps \a grpc_completion_queue_pluck. @@ -299,11 +299,9 @@ class ServerCompletionQueue : public CompletionQueue { /// AsyncNext()). By default all server completion queues are assumed to be /// frequently polled. ServerCompletionQueue(grpc_cq_polling_type polling_type) - : CompletionQueue(MakeCompletionQueue(polling_type)), + : CompletionQueue(grpc_completion_queue_attributes{ + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, polling_type}), polling_type_(polling_type) {} - - static grpc_completion_queue* MakeCompletionQueue( - grpc_cq_polling_type polling_type); }; } // namespace grpc diff --git a/include/grpc++/impl/codegen/core_codegen.h b/include/grpc++/impl/codegen/core_codegen.h index 65151590b25..3cb7da8ef6c 100644 --- a/include/grpc++/impl/codegen/core_codegen.h +++ b/include/grpc++/impl/codegen/core_codegen.h @@ -44,8 +44,15 @@ namespace grpc { /// Implementation of the core codegen interface. -class CoreCodegen : public CoreCodegenInterface { +class CoreCodegen final : public CoreCodegenInterface { private: + virtual const grpc_completion_queue_factory* + grpc_completion_queue_factory_lookup( + const grpc_completion_queue_attributes* attributes) override; + virtual grpc_completion_queue* grpc_completion_queue_create( + const grpc_completion_queue_factory* factory, + const grpc_completion_queue_attributes* attributes, + void* reserved) override; grpc_completion_queue* grpc_completion_queue_create_for_next( void* reserved) override; grpc_completion_queue* grpc_completion_queue_create_for_pluck( diff --git a/include/grpc++/impl/codegen/core_codegen_interface.h b/include/grpc++/impl/codegen/core_codegen_interface.h index 529bef687bb..a1a0aaf3cad 100644 --- a/include/grpc++/impl/codegen/core_codegen_interface.h +++ b/include/grpc++/impl/codegen/core_codegen_interface.h @@ -59,6 +59,12 @@ class CoreCodegenInterface { virtual void assert_fail(const char* failed_assertion, const char* file, int line) = 0; + virtual const grpc_completion_queue_factory* + grpc_completion_queue_factory_lookup( + const grpc_completion_queue_attributes* attributes) = 0; + virtual grpc_completion_queue* grpc_completion_queue_create( + const grpc_completion_queue_factory* factory, + const grpc_completion_queue_attributes* attributes, void* reserved) = 0; virtual grpc_completion_queue* grpc_completion_queue_create_for_next( void* reserved) = 0; virtual grpc_completion_queue* grpc_completion_queue_create_for_pluck( diff --git a/include/grpc++/impl/codegen/sync_stream.h b/include/grpc++/impl/codegen/sync_stream.h index 328d5cb1e83..a010924cefe 100644 --- a/include/grpc++/impl/codegen/sync_stream.h +++ b/include/grpc++/impl/codegen/sync_stream.h @@ -156,7 +156,9 @@ class ClientReader final : public ClientReaderInterface { ClientReader(ChannelInterface* channel, const RpcMethod& method, ClientContext* context, const W& request) : context_(context), - cq_(true), // Pluckable cq + cq_(grpc_completion_queue_attributes{ + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, + GRPC_CQ_DEFAULT_POLLING}), // Pluckable cq call_(channel->CreateCall(method, context, &cq_)) { CallOpSet @@ -230,7 +232,9 @@ class ClientWriter : public ClientWriterInterface { ClientWriter(ChannelInterface* channel, const RpcMethod& method, ClientContext* context, R* response) : context_(context), - cq_(true), // Pluckable cq + cq_(grpc_completion_queue_attributes{ + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, + GRPC_CQ_DEFAULT_POLLING}), // Pluckable cq call_(channel->CreateCall(method, context, &cq_)) { finish_ops_.RecvMessage(response); finish_ops_.AllowNoMessage(); @@ -330,7 +334,9 @@ class ClientReaderWriter final : public ClientReaderWriterInterface { ClientReaderWriter(ChannelInterface* channel, const RpcMethod& method, ClientContext* context) : context_(context), - cq_(true), // Pluckable cq + cq_(grpc_completion_queue_attributes{ + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, + GRPC_CQ_DEFAULT_POLLING}), // Pluckable cq call_(channel->CreateCall(method, context, &cq_)) { if (!context_->initial_metadata_corked_) { CallOpSet ops; diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index f3201edad29..1a7d0120bfa 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -93,29 +93,6 @@ GRPCAPI const char *grpc_version_string(void); /** Return a string specifying what the 'g' in gRPC stands for */ GRPCAPI const char *grpc_g_stands_for(void); -/** Specifies the type of APIs to use to pop events from the completion queue */ -typedef enum { - /** Events are popped out by calling grpc_completion_queue_next() API ONLY */ - GRPC_CQ_NEXT = 1, - - /** Events are popped out by calling grpc_completion_queue_pluck() API ONLY*/ - GRPC_CQ_PLUCK -} grpc_cq_completion_type; - -#define GRPC_CQ_CURRENT_VERSION 1 -typedef struct grpc_completion_queue_attributes { - /* The version number of this structure. More fields might be added to this - structure in future. */ - int version; /* Set to GRPC_CQ_CURRENT_VERSION */ - - grpc_cq_completion_type cq_completion_type; - - grpc_cq_polling_type cq_polling_type; -} grpc_completion_queue_attributes; - -/** The completion queue factory structure is opaque to the callers of grpc */ -typedef struct grpc_completion_queue_factory grpc_completion_queue_factory; - /** Returns the completion queue factory based on the attributes. MAY return a NULL if no factory can be found */ GRPCAPI const grpc_completion_queue_factory * diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 02732205f5f..a8eda739bf6 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -569,6 +569,29 @@ typedef enum { GRPC_CQ_NON_POLLING } grpc_cq_polling_type; +/** Specifies the type of APIs to use to pop events from the completion queue */ +typedef enum { + /** Events are popped out by calling grpc_completion_queue_next() API ONLY */ + GRPC_CQ_NEXT = 1, + + /** Events are popped out by calling grpc_completion_queue_pluck() API ONLY*/ + GRPC_CQ_PLUCK +} grpc_cq_completion_type; + +#define GRPC_CQ_CURRENT_VERSION 1 +typedef struct grpc_completion_queue_attributes { + /* The version number of this structure. More fields might be added to this + structure in future. */ + int version; /* Set to GRPC_CQ_CURRENT_VERSION */ + + grpc_cq_completion_type cq_completion_type; + + grpc_cq_polling_type cq_polling_type; +} grpc_completion_queue_attributes; + +/** The completion queue factory structure is opaque to the callers of grpc */ +typedef struct grpc_completion_queue_factory grpc_completion_queue_factory; + #ifdef __cplusplus } #endif diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index e17f094837e..ea97a6f374b 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -61,6 +61,7 @@ typedef struct { } plucker; typedef struct { + bool can_get_pollset; size_t (*size)(void); void (*init)(grpc_pollset *pollset, gpr_mu **mu); grpc_error *(*kick)(grpc_pollset *pollset, @@ -107,9 +108,10 @@ static grpc_error *non_polling_poller_work(grpc_exec_ctx *exec_ctx, gpr_timespec now, gpr_timespec deadline) { non_polling_poller *npp = (non_polling_poller *)pollset; + if (npp->shutdown) return GRPC_ERROR_NONE; non_polling_worker w; gpr_cv_init(&w.cv); - *worker = (grpc_pollset_worker *)&w; + if (worker != NULL) *worker = (grpc_pollset_worker *)&w; if (npp->root == NULL) { npp->root = w.next = w.prev = &w; } else { @@ -128,11 +130,11 @@ static grpc_error *non_polling_poller_work(grpc_exec_ctx *exec_ctx, } npp->root = NULL; } - w.next->prev = w.prev; - w.prev->next = w.next; } + w.next->prev = w.prev; + w.prev->next = w.next; gpr_cv_destroy(&w.cv); - *worker = NULL; + if (worker != NULL) *worker = NULL; return GRPC_ERROR_NONE; } @@ -169,21 +171,24 @@ static void non_polling_poller_shutdown(grpc_exec_ctx *exec_ctx, static const cq_poller_vtable g_poller_vtable_by_poller_type[] = { /* GRPC_CQ_DEFAULT_POLLING */ - {.size = grpc_pollset_size, + {.can_get_pollset = true, + .size = grpc_pollset_size, .init = grpc_pollset_init, .kick = grpc_pollset_kick, .work = grpc_pollset_work, .shutdown = grpc_pollset_shutdown, .destroy = grpc_pollset_destroy}, /* GRPC_CQ_NON_LISTENING */ - {.size = grpc_pollset_size, + {.can_get_pollset = true, + .size = grpc_pollset_size, .init = grpc_pollset_init, .kick = grpc_pollset_kick, .work = grpc_pollset_work, .shutdown = grpc_pollset_shutdown, .destroy = grpc_pollset_destroy}, /* GRPC_CQ_NON_POLLING */ - {.size = non_polling_poller_size, + {.can_get_pollset = false, + .size = non_polling_poller_size, .init = non_polling_poller_init, .kick = non_polling_poller_kick, .work = non_polling_poller_work, @@ -837,7 +842,7 @@ void grpc_completion_queue_destroy(grpc_completion_queue *cc) { } grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) { - return POLLSET_FROM_CQ(cc); + return cc->poller_vtable->can_get_pollset ? POLLSET_FROM_CQ(cc) : NULL; } grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps) { diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 9496f90390d..767c91a5ec5 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -1009,6 +1009,8 @@ void grpc_server_register_completion_queue(grpc_server *server, calls grpc_completion_queue_pluck() on server completion queues */ } + GPR_ASSERT(grpc_cq_pollset(cq)); + register_completion_queue(server, cq, false, reserved); } @@ -1102,8 +1104,9 @@ void grpc_server_start(grpc_server *server) { gpr_malloc(sizeof(*server->requested_calls_per_cq) * server->cq_count); for (i = 0; i < server->cq_count; i++) { if (!grpc_cq_is_non_listening_server_cq(server->cqs[i])) { - server->pollsets[server->pollset_count++] = - grpc_cq_pollset(server->cqs[i]); + grpc_pollset *pollset = grpc_cq_pollset(server->cqs[i]); + GPR_ASSERT(pollset); + server->pollsets[server->pollset_count++] = pollset; } server->request_freelist_per_cq[i] = gpr_stack_lockfree_create((size_t)server->max_requested_calls_per_cq); diff --git a/src/cpp/common/core_codegen.cc b/src/cpp/common/core_codegen.cc index 0dd758ec4e5..8f1de222fb8 100644 --- a/src/cpp/common/core_codegen.cc +++ b/src/cpp/common/core_codegen.cc @@ -54,6 +54,18 @@ struct grpc_byte_buffer; namespace grpc { +const grpc_completion_queue_factory* +CoreCodegen::grpc_completion_queue_factory_lookup( + const grpc_completion_queue_attributes* attributes) { + return ::grpc_completion_queue_factory_lookup(attributes); +} + +grpc_completion_queue* CoreCodegen::grpc_completion_queue_create( + const grpc_completion_queue_factory* factory, + const grpc_completion_queue_attributes* attributes, void* reserved) { + return ::grpc_completion_queue_create(factory, attributes, reserved); +} + grpc_completion_queue* CoreCodegen::grpc_completion_queue_create_for_next( void* reserved) { return ::grpc_completion_queue_create_for_next(reserved); diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc index c6784ea1593..6687fe78b92 100644 --- a/src/cpp/server/server_builder.cc +++ b/src/cpp/server/server_builder.cc @@ -243,6 +243,16 @@ std::unique_ptr ServerBuilder::BuildAndStart() { sync_server_cqs(std::make_shared< std::vector>>()); + int num_frequently_polled_cqs = 0; + for (auto it = cqs_.begin(); it != cqs_.end(); ++it) { + if ((*it)->IsFrequentlyPolled()) { + num_frequently_polled_cqs++; + } + } + + const bool is_hybrid_server = + has_sync_methods && num_frequently_polled_cqs > 0; + if (has_sync_methods) { // This is a Sync server gpr_log(GPR_INFO, @@ -253,7 +263,7 @@ std::unique_ptr ServerBuilder::BuildAndStart() { sync_server_settings_.cq_timeout_msec); grpc_cq_polling_type polling_type = - cqs_.empty() ? GRPC_CQ_DEFAULT_POLLING : GRPC_CQ_NON_POLLING; + is_hybrid_server ? GRPC_CQ_NON_POLLING : GRPC_CQ_DEFAULT_POLLING; // Create completion queues to listen to incoming rpc requests for (int i = 0; i < sync_server_settings_.num_cqs; i++) { @@ -273,12 +283,15 @@ std::unique_ptr ServerBuilder::BuildAndStart() { // server // 2. cqs_: Completion queues added via AddCompletionQueue() call - // All sync cqs (if any) are frequently polled by ThreadManager - int num_frequently_polled_cqs = sync_server_cqs->size(); - for (auto it = sync_server_cqs->begin(); it != sync_server_cqs->end(); ++it) { - grpc_server_register_completion_queue(server->server_, (*it)->cq(), - nullptr); + if (is_hybrid_server) { + grpc_server_register_non_listening_completion_queue(server->server_, + (*it)->cq(), nullptr); + } else { + grpc_server_register_completion_queue(server->server_, (*it)->cq(), + nullptr); + } + num_frequently_polled_cqs++; } // cqs_ contains the completion queue added by calling the ServerBuilder's @@ -290,7 +303,6 @@ std::unique_ptr ServerBuilder::BuildAndStart() { if ((*it)->IsFrequentlyPolled()) { grpc_server_register_completion_queue(server->server_, (*it)->cq(), nullptr); - num_frequently_polled_cqs++; } else { grpc_server_register_non_listening_completion_queue(server->server_, (*it)->cq(), nullptr); diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 0b5215ef8e4..cc3958bf13c 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,7 @@ #include #include "src/core/lib/iomgr/port.h" +#include "src/proto/grpc/health/v1/health.grpc.pb.h" #include "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "test/core/util/port.h" @@ -224,13 +226,15 @@ class ServerBuilderSyncPluginDisabler : public ::grpc::ServerBuilderOption { class TestScenario { public: - TestScenario(bool non_block, const grpc::string& creds_type, + TestScenario(bool non_block, const grpc::string& creds_type, bool hcs, const grpc::string& content) : disable_blocking(non_block), + health_check_service(hcs), credentials_type(creds_type), message_content(content) {} void Log() const; bool disable_blocking; + bool health_check_service; // Although the below grpc::string's are logically const, we can't declare // them const because of a limitation in the way old compilers (e.g., gcc-4.4) // manage vector insertion using a copy constructor @@ -243,6 +247,8 @@ static std::ostream& operator<<(std::ostream& out, return out << "TestScenario{disable_blocking=" << (scenario.disable_blocking ? "true" : "false") << ", credentials='" << scenario.credentials_type + << ", health_check_service=" + << (scenario.health_check_service ? "true" : "false") << "', message_size=" << scenario.message_content.size() << "}"; } @@ -252,6 +258,8 @@ void TestScenario::Log() const { gpr_log(GPR_DEBUG, "%s", out.str().c_str()); } +class HealthCheck : public health::v1::Health::Service {}; + class AsyncEnd2endTest : public ::testing::TestWithParam { protected: AsyncEnd2endTest() { GetParam().Log(); } @@ -268,6 +276,9 @@ class AsyncEnd2endTest : public ::testing::TestWithParam { GetParam().credentials_type); builder.AddListeningPort(server_address_.str(), server_creds); builder.RegisterService(&service_); + if (GetParam().health_check_service) { + builder.RegisterService(&health_check_); + } cq_ = builder.AddCompletionQueue(); // TODO(zyc): make a test option to choose wheather sync plugins should be @@ -340,6 +351,7 @@ class AsyncEnd2endTest : public ::testing::TestWithParam { std::unique_ptr stub_; std::unique_ptr server_; grpc::testing::EchoTestService::AsyncService service_; + HealthCheck health_check_; std::ostringstream server_address_; int port_; @@ -1754,12 +1766,14 @@ std::vector CreateTestScenarios(bool test_disable_blocking, messages.push_back(big_msg); } - for (auto cred = credentials_types.begin(); cred != credentials_types.end(); - ++cred) { - for (auto msg = messages.begin(); msg != messages.end(); msg++) { - scenarios.emplace_back(false, *cred, *msg); - if (test_disable_blocking) { - scenarios.emplace_back(true, *cred, *msg); + for (auto health_check_service : {false, true}) { + for (auto cred = credentials_types.begin(); cred != credentials_types.end(); + ++cred) { + for (auto msg = messages.begin(); msg != messages.end(); msg++) { + scenarios.emplace_back(false, *cred, health_check_service, *msg); + if (test_disable_blocking) { + scenarios.emplace_back(true, *cred, health_check_service, *msg); + } } } } From 58aa706aaf1c39e092f246202b18e6a2931dc664 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 08:10:58 -0700 Subject: [PATCH 11/51] Fix registration --- src/core/lib/surface/server.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 767c91a5ec5..1680085f67b 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -1009,8 +1009,6 @@ void grpc_server_register_completion_queue(grpc_server *server, calls grpc_completion_queue_pluck() on server completion queues */ } - GPR_ASSERT(grpc_cq_pollset(cq)); - register_completion_queue(server, cq, false, reserved); } @@ -1105,8 +1103,7 @@ void grpc_server_start(grpc_server *server) { for (i = 0; i < server->cq_count; i++) { if (!grpc_cq_is_non_listening_server_cq(server->cqs[i])) { grpc_pollset *pollset = grpc_cq_pollset(server->cqs[i]); - GPR_ASSERT(pollset); - server->pollsets[server->pollset_count++] = pollset; + if (pollset != NULL) server->pollsets[server->pollset_count++] = pollset; } server->request_freelist_per_cq[i] = gpr_stack_lockfree_create((size_t)server->max_requested_calls_per_cq); From 11c5832b3e8be35f16465d8ef38a1d0ba033a822 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 08:21:17 -0700 Subject: [PATCH 12/51] Get rid of second api for marking non-listening cqs --- include/grpc/grpc.h | 9 --------- src/core/lib/surface/completion_queue.c | 12 +++++++++++- src/core/lib/surface/completion_queue.h | 5 ++--- src/core/lib/surface/server.c | 23 +++++------------------ src/cpp/server/server_builder.cc | 18 ++++-------------- 5 files changed, 22 insertions(+), 45 deletions(-) diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 1a7d0120bfa..7b37f34acc4 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -376,15 +376,6 @@ GRPCAPI void grpc_server_register_completion_queue(grpc_server *server, grpc_completion_queue *cq, void *reserved); -/** Register a non-listening completion queue with the server. This API is - similar to grpc_server_register_completion_queue except that the server will - not use this completion_queue to listen to any incoming channels. - - Registering a non-listening completion queue will have negative performance - impact and hence this API is not recommended for production use cases. */ -GRPCAPI void grpc_server_register_non_listening_completion_queue( - grpc_server *server, grpc_completion_queue *q, void *reserved); - /** Add a HTTP2 over plaintext over tcp listener. Returns bound port number on success, 0 on failure. REQUIRES: server not started */ diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index ea97a6f374b..eae3f103b12 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -62,6 +62,7 @@ typedef struct { typedef struct { bool can_get_pollset; + bool can_listen; size_t (*size)(void); void (*init)(grpc_pollset *pollset, gpr_mu **mu); grpc_error *(*kick)(grpc_pollset *pollset, @@ -172,6 +173,7 @@ static void non_polling_poller_shutdown(grpc_exec_ctx *exec_ctx, static const cq_poller_vtable g_poller_vtable_by_poller_type[] = { /* GRPC_CQ_DEFAULT_POLLING */ {.can_get_pollset = true, + .can_listen = true, .size = grpc_pollset_size, .init = grpc_pollset_init, .kick = grpc_pollset_kick, @@ -180,6 +182,7 @@ static const cq_poller_vtable g_poller_vtable_by_poller_type[] = { .destroy = grpc_pollset_destroy}, /* GRPC_CQ_NON_LISTENING */ {.can_get_pollset = true, + .can_listen = false, .size = grpc_pollset_size, .init = grpc_pollset_init, .kick = grpc_pollset_kick, @@ -188,6 +191,7 @@ static const cq_poller_vtable g_poller_vtable_by_poller_type[] = { .destroy = grpc_pollset_destroy}, /* GRPC_CQ_NON_POLLING */ {.can_get_pollset = false, + .can_listen = false, .size = non_polling_poller_size, .init = non_polling_poller_init, .kick = non_polling_poller_kick, @@ -863,4 +867,10 @@ bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc) { void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; } -int grpc_cq_is_server_cq(grpc_completion_queue *cc) { return cc->is_server_cq; } +bool grpc_cq_is_server_cq(grpc_completion_queue *cc) { + return cc->is_server_cq; +} + +bool grpc_cq_can_listen(grpc_completion_queue *cc) { + return cc->poller_vtable->can_listen; +} diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index 0995a56889b..a932087939d 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -94,10 +94,9 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc, grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc); grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps); -void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc); -bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc); void grpc_cq_mark_server_cq(grpc_completion_queue *cc); -int grpc_cq_is_server_cq(grpc_completion_queue *cc); +bool grpc_cq_is_server_cq(grpc_completion_queue *cc); +bool grpc_cq_can_listen(grpc_completion_queue *cc); grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue *cc); diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 1680085f67b..da8b6339b2b 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -974,7 +974,7 @@ const grpc_channel_filter grpc_server_top_filter = { static void register_completion_queue(grpc_server *server, grpc_completion_queue *cq, - bool is_non_listening, void *reserved) { + void *reserved) { size_t i, n; GPR_ASSERT(!reserved); for (i = 0; i < server->cq_count; i++) { @@ -983,10 +983,6 @@ static void register_completion_queue(grpc_server *server, grpc_cq_mark_server_cq(cq); - if (is_non_listening) { - grpc_cq_mark_non_listening_server_cq(cq); - } - GRPC_CQ_INTERNAL_REF(cq, "server"); n = server->cq_count++; server->cqs = gpr_realloc(server->cqs, @@ -1009,16 +1005,7 @@ void grpc_server_register_completion_queue(grpc_server *server, calls grpc_completion_queue_pluck() on server completion queues */ } - register_completion_queue(server, cq, false, reserved); -} - -void grpc_server_register_non_listening_completion_queue( - grpc_server *server, grpc_completion_queue *cq, void *reserved) { - GRPC_API_TRACE( - "grpc_server_register_non_listening_completion_queue(server=%p, cq=%p, " - "reserved=%p)", - 3, (server, cq, reserved)); - register_completion_queue(server, cq, true, reserved); + register_completion_queue(server, cq, reserved); } grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) { @@ -1101,9 +1088,9 @@ void grpc_server_start(grpc_server *server) { server->requested_calls_per_cq = gpr_malloc(sizeof(*server->requested_calls_per_cq) * server->cq_count); for (i = 0; i < server->cq_count; i++) { - if (!grpc_cq_is_non_listening_server_cq(server->cqs[i])) { - grpc_pollset *pollset = grpc_cq_pollset(server->cqs[i]); - if (pollset != NULL) server->pollsets[server->pollset_count++] = pollset; + if (grpc_cq_can_listen(server->cqs[i])) { + server->pollsets[server->pollset_count++] = + grpc_cq_pollset(server->cqs[i]); } server->request_freelist_per_cq[i] = gpr_stack_lockfree_create((size_t)server->max_requested_calls_per_cq); diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc index 6687fe78b92..a92cec643ce 100644 --- a/src/cpp/server/server_builder.cc +++ b/src/cpp/server/server_builder.cc @@ -284,13 +284,8 @@ std::unique_ptr ServerBuilder::BuildAndStart() { // 2. cqs_: Completion queues added via AddCompletionQueue() call for (auto it = sync_server_cqs->begin(); it != sync_server_cqs->end(); ++it) { - if (is_hybrid_server) { - grpc_server_register_non_listening_completion_queue(server->server_, - (*it)->cq(), nullptr); - } else { - grpc_server_register_completion_queue(server->server_, (*it)->cq(), - nullptr); - } + grpc_server_register_completion_queue(server->server_, (*it)->cq(), + nullptr); num_frequently_polled_cqs++; } @@ -300,13 +295,8 @@ std::unique_ptr ServerBuilder::BuildAndStart() { // listening to incoming channels. Such completion queues must be registered // as non-listening queues for (auto it = cqs_.begin(); it != cqs_.end(); ++it) { - if ((*it)->IsFrequentlyPolled()) { - grpc_server_register_completion_queue(server->server_, (*it)->cq(), - nullptr); - } else { - grpc_server_register_non_listening_completion_queue(server->server_, - (*it)->cq(), nullptr); - } + grpc_server_register_completion_queue(server->server_, (*it)->cq(), + nullptr); } if (num_frequently_polled_cqs == 0) { From b3612d3cd409c421065e9faf53dae507f5fc6d7b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 14:02:13 -0700 Subject: [PATCH 13/51] Remove API --- grpc.def | 1 - include/grpc/grpc.h | 9 --------- src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 -- src/ruby/ext/grpc/rb_grpc_imports.generated.h | 3 --- 4 files changed, 15 deletions(-) diff --git a/grpc.def b/grpc.def index 1589316a588..da42d736980 100644 --- a/grpc.def +++ b/grpc.def @@ -88,7 +88,6 @@ EXPORTS grpc_server_request_registered_call grpc_server_create grpc_server_register_completion_queue - grpc_server_register_non_listening_completion_queue grpc_server_add_insecure_http2_port grpc_server_start grpc_server_shutdown_and_notify diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index f3201edad29..4da43706655 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -399,15 +399,6 @@ GRPCAPI void grpc_server_register_completion_queue(grpc_server *server, grpc_completion_queue *cq, void *reserved); -/** Register a non-listening completion queue with the server. This API is - similar to grpc_server_register_completion_queue except that the server will - not use this completion_queue to listen to any incoming channels. - - Registering a non-listening completion queue will have negative performance - impact and hence this API is not recommended for production use cases. */ -GRPCAPI void grpc_server_register_non_listening_completion_queue( - grpc_server *server, grpc_completion_queue *q, void *reserved); - /** Add a HTTP2 over plaintext over tcp listener. Returns bound port number on success, 0 on failure. REQUIRES: server not started */ diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 063f92114c0..fd4fb9ea4d3 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -126,7 +126,6 @@ grpc_server_register_method_type grpc_server_register_method_import; grpc_server_request_registered_call_type grpc_server_request_registered_call_import; grpc_server_create_type grpc_server_create_import; grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import; -grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import; grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import; grpc_server_start_type grpc_server_start_import; grpc_server_shutdown_and_notify_type grpc_server_shutdown_and_notify_import; @@ -423,7 +422,6 @@ void grpc_rb_load_imports(HMODULE library) { grpc_server_request_registered_call_import = (grpc_server_request_registered_call_type) GetProcAddress(library, "grpc_server_request_registered_call"); grpc_server_create_import = (grpc_server_create_type) GetProcAddress(library, "grpc_server_create"); grpc_server_register_completion_queue_import = (grpc_server_register_completion_queue_type) GetProcAddress(library, "grpc_server_register_completion_queue"); - grpc_server_register_non_listening_completion_queue_import = (grpc_server_register_non_listening_completion_queue_type) GetProcAddress(library, "grpc_server_register_non_listening_completion_queue"); grpc_server_add_insecure_http2_port_import = (grpc_server_add_insecure_http2_port_type) GetProcAddress(library, "grpc_server_add_insecure_http2_port"); grpc_server_start_import = (grpc_server_start_type) GetProcAddress(library, "grpc_server_start"); grpc_server_shutdown_and_notify_import = (grpc_server_shutdown_and_notify_type) GetProcAddress(library, "grpc_server_shutdown_and_notify"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index e036ff7bd63..7289ae3a352 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -329,9 +329,6 @@ extern grpc_server_create_type grpc_server_create_import; typedef void(*grpc_server_register_completion_queue_type)(grpc_server *server, grpc_completion_queue *cq, void *reserved); extern grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import; #define grpc_server_register_completion_queue grpc_server_register_completion_queue_import -typedef void(*grpc_server_register_non_listening_completion_queue_type)(grpc_server *server, grpc_completion_queue *q, void *reserved); -extern grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import; -#define grpc_server_register_non_listening_completion_queue grpc_server_register_non_listening_completion_queue_import typedef int(*grpc_server_add_insecure_http2_port_type)(grpc_server *server, const char *addr); extern grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import; #define grpc_server_add_insecure_http2_port grpc_server_add_insecure_http2_port_import From cd3ae4f33b6d657b211d3cc186120dd3a297d212 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 14:19:23 -0700 Subject: [PATCH 14/51] Remove API --- src/node/ext/server_generic.cc | 8 +++++--- src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi | 3 --- src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi | 11 +---------- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/node/ext/server_generic.cc b/src/node/ext/server_generic.cc index 24573bd52f5..a25b2f1ca74 100644 --- a/src/node/ext/server_generic.cc +++ b/src/node/ext/server_generic.cc @@ -44,9 +44,11 @@ namespace grpc { namespace node { Server::Server(grpc_server *server) : wrapped_server(server) { - shutdown_queue = grpc_completion_queue_create_for_pluck(NULL); - grpc_server_register_non_listening_completion_queue(server, shutdown_queue, - NULL); + grpc_completion_queue_attributes attrs = { + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, GRPC_CQ_NON_LISTENING}; + shutdown_queue = grpc_completion_queue_create( + grpc_completion_queue_factory_lookup(&attrs), &attrs, NULL); + grpc_server_completion_queue(server, shutdown_queue, NULL); } Server::~Server() { diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi index 0b2bdef48be..74e4bc6a69c 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi @@ -356,8 +356,6 @@ cdef extern from "grpc/grpc.h": void grpc_server_register_completion_queue(grpc_server *server, grpc_completion_queue *cq, void *reserved) nogil - void grpc_server_register_non_listening_completion_queue( - grpc_server *server, grpc_completion_queue *cq, void *reserved) nogil int grpc_server_add_insecure_http2_port( grpc_server *server, const char *addr) nogil void grpc_server_start(grpc_server *server) nogil @@ -502,4 +500,3 @@ cdef extern from "grpc/compression.h": int grpc_compression_options_is_algorithm_enabled( const grpc_compression_options *opts, grpc_compression_algorithm algorithm) nogil - diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi index 18db38b6861..97192efda74 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi @@ -82,20 +82,11 @@ cdef class Server: self.c_server, queue.c_completion_queue, NULL) self.registered_completion_queues.append(queue) - def register_non_listening_completion_queue( - self, CompletionQueue queue not None): - if self.is_started: - raise ValueError("cannot register completion queues after start") - with nogil: - grpc_server_register_non_listening_completion_queue( - self.c_server, queue.c_completion_queue, NULL) - self.registered_completion_queues.append(queue) - def start(self): if self.is_started: raise ValueError("the server has already started") self.backup_shutdown_queue = CompletionQueue() - self.register_non_listening_completion_queue(self.backup_shutdown_queue) + self.register_completion_queue(self.backup_shutdown_queue) self.is_started = True with nogil: grpc_server_start(self.c_server) From 3512ec926b09099b30ea7b496c15dcb559ace7a3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 18 Apr 2017 13:26:32 -0700 Subject: [PATCH 15/51] Fix typo --- src/node/ext/server_generic.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/ext/server_generic.cc b/src/node/ext/server_generic.cc index a25b2f1ca74..088273d527c 100644 --- a/src/node/ext/server_generic.cc +++ b/src/node/ext/server_generic.cc @@ -48,7 +48,7 @@ Server::Server(grpc_server *server) : wrapped_server(server) { GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, GRPC_CQ_NON_LISTENING}; shutdown_queue = grpc_completion_queue_create( grpc_completion_queue_factory_lookup(&attrs), &attrs, NULL); - grpc_server_completion_queue(server, shutdown_queue, NULL); + grpc_server_register_completion_queue(server, shutdown_queue, NULL); } Server::~Server() { From 0a458b599e723bfcd22cf92755059ac4de0a1489 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 19 Apr 2017 06:35:16 -0700 Subject: [PATCH 16/51] Fix Bazel build --- src/proto/grpc/health/v1/BUILD | 39 ++++++++++++++++++++++++++++++++++ test/cpp/end2end/BUILD | 1 + 2 files changed, 40 insertions(+) create mode 100644 src/proto/grpc/health/v1/BUILD diff --git a/src/proto/grpc/health/v1/BUILD b/src/proto/grpc/health/v1/BUILD new file mode 100644 index 00000000000..dbb91d91392 --- /dev/null +++ b/src/proto/grpc/health/v1/BUILD @@ -0,0 +1,39 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +licenses(["notice"]) # 3-clause BSD + +package(default_visibility = ["//visibility:public"]) + +load("//bazel:grpc_build_system.bzl", "grpc_proto_library") + +grpc_proto_library( + name = "health_proto", + srcs = ["health.proto"], +) diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD index 0bf7948fcfe..a74a123aef3 100644 --- a/test/cpp/end2end/BUILD +++ b/test/cpp/end2end/BUILD @@ -156,6 +156,7 @@ cc_test( "//src/proto/grpc/testing:echo_messages_proto", "//src/proto/grpc/testing:echo_proto", "//src/proto/grpc/testing/duplicate:echo_duplicate_proto", + "//src/proto/grpc/health/v1:health_proto", "//test/core/util:gpr_test_util", "//test/core/util:grpc_test_util", "//test/cpp/util:test_util", From e300670153d3c7cf62660c49c90cbfc6a63f3c0c Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 19 Apr 2017 07:43:56 -0700 Subject: [PATCH 17/51] Simplify hash table API: remove per-entry vtable and take ownership of keys and values. --- .../filters/client_channel/client_channel.c | 9 +------- .../client_channel/lb_policy/grpclb/grpclb.c | 16 +++----------- .../message_size/message_size_filter.c | 11 +--------- src/core/lib/slice/slice_hash_table.c | 21 ++++++++++--------- src/core/lib/slice/slice_hash_table.h | 15 +++++-------- src/core/lib/transport/service_config.c | 21 +++++++------------ src/core/lib/transport/service_config.h | 4 ++-- 7 files changed, 31 insertions(+), 66 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index ce9abdad610..8d28e829d81 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -96,17 +96,10 @@ static void method_parameters_unref(method_parameters *method_params) { } } -static void *method_parameters_copy(void *value) { - return method_parameters_ref(value); -} - static void method_parameters_free(grpc_exec_ctx *exec_ctx, void *value) { method_parameters_unref(value); } -static const grpc_slice_hash_table_vtable method_parameters_vtable = { - method_parameters_free, method_parameters_copy}; - static bool parse_wait_for_ready(grpc_json *field, wait_for_ready_value *wait_for_ready) { if (field->type != GRPC_JSON_TRUE && field->type != GRPC_JSON_FALSE) { @@ -472,7 +465,7 @@ static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, grpc_uri_destroy(uri); method_params_table = grpc_service_config_create_method_config_table( exec_ctx, service_config, method_parameters_create_from_json, - &method_parameters_vtable); + method_parameters_free); grpc_service_config_destroy(service_config); } } diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index ff8d3193098..3fe3f056d33 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -750,18 +750,11 @@ static void destroy_balancer_name(grpc_exec_ctx *exec_ctx, gpr_free(balancer_name); } -static void *copy_balancer_name(void *balancer_name) { - return gpr_strdup(balancer_name); -} - static grpc_slice_hash_table_entry targets_info_entry_create( const char *address, const char *balancer_name) { - static const grpc_slice_hash_table_vtable vtable = {destroy_balancer_name, - copy_balancer_name}; grpc_slice_hash_table_entry entry; entry.key = grpc_slice_from_copied_string(address); - entry.value = (void *)balancer_name; - entry.vtable = &vtable; + entry.value = gpr_strdup(balancer_name); return entry; } @@ -825,11 +818,8 @@ static char *get_lb_uri_target_addresses(grpc_exec_ctx *exec_ctx, uri_path); gpr_free(uri_path); - *targets_info = - grpc_slice_hash_table_create(num_grpclb_addrs, targets_info_entries); - for (size_t i = 0; i < num_grpclb_addrs; i++) { - grpc_slice_unref_internal(exec_ctx, targets_info_entries[i].key); - } + *targets_info = grpc_slice_hash_table_create( + num_grpclb_addrs, targets_info_entries, destroy_balancer_name); gpr_free(targets_info_entries); return target_uri_str; diff --git a/src/core/ext/filters/message_size/message_size_filter.c b/src/core/ext/filters/message_size/message_size_filter.c index db0f0119053..69ebbc74149 100644 --- a/src/core/ext/filters/message_size/message_size_filter.c +++ b/src/core/ext/filters/message_size/message_size_filter.c @@ -50,19 +50,10 @@ typedef struct message_size_limits { int max_recv_size; } message_size_limits; -static void* message_size_limits_copy(void* value) { - void* new_value = gpr_malloc(sizeof(message_size_limits)); - memcpy(new_value, value, sizeof(message_size_limits)); - return new_value; -} - static void message_size_limits_free(grpc_exec_ctx* exec_ctx, void* value) { gpr_free(value); } -static const grpc_slice_hash_table_vtable message_size_limits_vtable = { - message_size_limits_free, message_size_limits_copy}; - static void* message_size_limits_create_from_json(const grpc_json* json) { int max_request_message_bytes = -1; int max_response_message_bytes = -1; @@ -255,7 +246,7 @@ static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx, chand->method_limit_table = grpc_service_config_create_method_config_table( exec_ctx, service_config, message_size_limits_create_from_json, - &message_size_limits_vtable); + message_size_limits_free); grpc_service_config_destroy(service_config); } } diff --git a/src/core/lib/slice/slice_hash_table.c b/src/core/lib/slice/slice_hash_table.c index 219567f36f8..7a8b79cc336 100644 --- a/src/core/lib/slice/slice_hash_table.c +++ b/src/core/lib/slice/slice_hash_table.c @@ -42,12 +42,13 @@ struct grpc_slice_hash_table { gpr_refcount refs; + void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value); size_t size; grpc_slice_hash_table_entry* entries; }; static bool is_empty(grpc_slice_hash_table_entry* entry) { - return entry->vtable == NULL; + return entry->value == NULL; } // Helper function for insert and get operations that performs quadratic @@ -67,23 +68,23 @@ static size_t grpc_slice_hash_table_find_index( return table->size; // Not found. } -static void grpc_slice_hash_table_add( - grpc_slice_hash_table* table, grpc_slice key, void* value, - const grpc_slice_hash_table_vtable* vtable) { +static void grpc_slice_hash_table_add(grpc_slice_hash_table* table, + grpc_slice key, void* value) { GPR_ASSERT(value != NULL); const size_t idx = grpc_slice_hash_table_find_index(table, key, true /* find_empty */); GPR_ASSERT(idx != table->size); // Table should never be full. grpc_slice_hash_table_entry* entry = &table->entries[idx]; - entry->key = grpc_slice_ref_internal(key); - entry->value = vtable->copy_value(value); - entry->vtable = vtable; + entry->key = key; + entry->value = value; } grpc_slice_hash_table* grpc_slice_hash_table_create( - size_t num_entries, grpc_slice_hash_table_entry* entries) { + size_t num_entries, grpc_slice_hash_table_entry* entries, + void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value)) { grpc_slice_hash_table* table = gpr_zalloc(sizeof(*table)); gpr_ref_init(&table->refs, 1); + table->destroy_value = destroy_value; // Quadratic probing gets best performance when the table is no more // than half full. table->size = num_entries * 2; @@ -91,7 +92,7 @@ grpc_slice_hash_table* grpc_slice_hash_table_create( table->entries = gpr_zalloc(entry_size); for (size_t i = 0; i < num_entries; ++i) { grpc_slice_hash_table_entry* entry = &entries[i]; - grpc_slice_hash_table_add(table, entry->key, entry->value, entry->vtable); + grpc_slice_hash_table_add(table, entry->key, entry->value); } return table; } @@ -108,7 +109,7 @@ void grpc_slice_hash_table_unref(grpc_exec_ctx* exec_ctx, grpc_slice_hash_table_entry* entry = &table->entries[i]; if (!is_empty(entry)) { grpc_slice_unref_internal(exec_ctx, entry->key); - entry->vtable->destroy_value(exec_ctx, entry->value); + table->destroy_value(exec_ctx, entry->value); } } gpr_free(table->entries); diff --git a/src/core/lib/slice/slice_hash_table.h b/src/core/lib/slice/slice_hash_table.h index d0c27122d7f..81e92b6e21f 100644 --- a/src/core/lib/slice/slice_hash_table.h +++ b/src/core/lib/slice/slice_hash_table.h @@ -41,29 +41,24 @@ * probing (https://en.wikipedia.org/wiki/Quadratic_probing). * * The keys are \a grpc_slice objects. The values are arbitrary pointers - * with a common vtable. + * with a common destroy function. * * Hash tables are intentionally immutable, to avoid the need for locking. */ typedef struct grpc_slice_hash_table grpc_slice_hash_table; -typedef struct grpc_slice_hash_table_vtable { - void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value); - void *(*copy_value)(void *value); -} grpc_slice_hash_table_vtable; - typedef struct grpc_slice_hash_table_entry { grpc_slice key; void *value; /* Must not be NULL. */ - const grpc_slice_hash_table_vtable *vtable; } grpc_slice_hash_table_entry; /** Creates a new hash table of containing \a entries, which is an array - of length \a num_entries. - Creates its own copy of all keys and values from \a entries. */ + of length \a num_entries. Takes ownership of all keys and values in + \a entries. Values will be cleaned up via \a destroy_value(). */ grpc_slice_hash_table *grpc_slice_hash_table_create( - size_t num_entries, grpc_slice_hash_table_entry *entries); + size_t num_entries, grpc_slice_hash_table_entry *entries, + void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value)); grpc_slice_hash_table *grpc_slice_hash_table_ref(grpc_slice_hash_table *table); void grpc_slice_hash_table_unref(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/transport/service_config.c b/src/core/lib/transport/service_config.c index 1195f75044a..3cda394723d 100644 --- a/src/core/lib/transport/service_config.c +++ b/src/core/lib/transport/service_config.c @@ -162,7 +162,7 @@ static char* parse_json_method_name(grpc_json* json) { static bool parse_json_method_config( grpc_exec_ctx* exec_ctx, grpc_json* json, void* (*create_value)(const grpc_json* method_config_json), - const grpc_slice_hash_table_vtable* vtable, + void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value), grpc_slice_hash_table_entry* entries, size_t* idx) { // Construct value. void* method_config = create_value(json); @@ -185,13 +185,12 @@ static bool parse_json_method_config( // Add entry for each path. for (size_t i = 0; i < paths.count; ++i) { entries[*idx].key = grpc_slice_from_copied_string(paths.strs[i]); - entries[*idx].value = vtable->copy_value(method_config); - entries[*idx].vtable = vtable; + entries[*idx].value = method_config; ++*idx; } success = true; done: - vtable->destroy_value(exec_ctx, method_config); + destroy_value(exec_ctx, method_config); gpr_strvec_destroy(&paths); return success; } @@ -199,7 +198,7 @@ done: grpc_slice_hash_table* grpc_service_config_create_method_config_table( grpc_exec_ctx* exec_ctx, const grpc_service_config* service_config, void* (*create_value)(const grpc_json* method_config_json), - const grpc_slice_hash_table_vtable* vtable) { + void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value)) { const grpc_json* json = service_config->json_tree; // Traverse parsed JSON tree. if (json->type != GRPC_JSON_OBJECT || json->key != NULL) return NULL; @@ -220,8 +219,8 @@ grpc_slice_hash_table* grpc_service_config_create_method_config_table( size_t idx = 0; for (grpc_json* method = field->child; method != NULL; method = method->next) { - if (!parse_json_method_config(exec_ctx, method, create_value, vtable, - entries, &idx)) { + if (!parse_json_method_config(exec_ctx, method, create_value, + destroy_value, entries, &idx)) { return NULL; } } @@ -231,12 +230,8 @@ grpc_slice_hash_table* grpc_service_config_create_method_config_table( // Instantiate method config table. grpc_slice_hash_table* method_config_table = NULL; if (entries != NULL) { - method_config_table = grpc_slice_hash_table_create(num_entries, entries); - // Clean up. - for (size_t i = 0; i < num_entries; ++i) { - grpc_slice_unref_internal(exec_ctx, entries[i].key); - vtable->destroy_value(exec_ctx, entries[i].value); - } + method_config_table = + grpc_slice_hash_table_create(num_entries, entries, destroy_value); gpr_free(entries); } return method_config_table; diff --git a/src/core/lib/transport/service_config.h b/src/core/lib/transport/service_config.h index ebfc59b5347..e0548b9c3fa 100644 --- a/src/core/lib/transport/service_config.h +++ b/src/core/lib/transport/service_config.h @@ -57,12 +57,12 @@ const char* grpc_service_config_get_lb_policy_name( /// Creates a method config table based on the data in \a json. /// The table's keys are request paths. The table's value type is /// returned by \a create_value(), based on data parsed from the JSON tree. -/// \a vtable provides methods used to manage the values. +/// \a destroy_value is used to clean up values. /// Returns NULL on error. grpc_slice_hash_table* grpc_service_config_create_method_config_table( grpc_exec_ctx* exec_ctx, const grpc_service_config* service_config, void* (*create_value)(const grpc_json* method_config_json), - const grpc_slice_hash_table_vtable* vtable); + void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value)); /// A helper function for looking up values in the table returned by /// \a grpc_service_config_create_method_config_table(). From ae1e29eed96798cadfed15b2e1e63e92dcf6d36e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 19 Apr 2017 09:07:23 -0700 Subject: [PATCH 18/51] Get dep in the right place --- test/cpp/end2end/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/end2end/BUILD b/test/cpp/end2end/BUILD index a74a123aef3..f1212e15c77 100644 --- a/test/cpp/end2end/BUILD +++ b/test/cpp/end2end/BUILD @@ -51,6 +51,7 @@ cc_test( "//src/proto/grpc/testing:echo_messages_proto", "//src/proto/grpc/testing:echo_proto", "//src/proto/grpc/testing/duplicate:echo_duplicate_proto", + "//src/proto/grpc/health/v1:health_proto", "//test/core/util:gpr_test_util", "//test/core/util:grpc_test_util", "//test/cpp/util:test_util", @@ -156,7 +157,6 @@ cc_test( "//src/proto/grpc/testing:echo_messages_proto", "//src/proto/grpc/testing:echo_proto", "//src/proto/grpc/testing/duplicate:echo_duplicate_proto", - "//src/proto/grpc/health/v1:health_proto", "//test/core/util:gpr_test_util", "//test/core/util:grpc_test_util", "//test/cpp/util:test_util", From 0c0b89a88ba6544e42cca43913ed65dea64bff3a Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 19 Apr 2017 13:28:24 -0700 Subject: [PATCH 19/51] Change hash table to use linear probing and add unit test. Also add some missing rules in test/core/slice/BUILD. --- CMakeLists.txt | 32 +++ Makefile | 36 ++++ build.yaml | 10 + src/core/lib/slice/slice_hash_table.c | 55 +++-- src/core/lib/slice/slice_hash_table.h | 4 +- test/core/slice/BUILD | 23 +- test/core/slice/slice_hash_table_test.c | 202 ++++++++++++++++++ .../generated/sources_and_headers.json | 17 ++ tools/run_tests/generated/tests.json | 22 ++ vsprojects/buildtests_c.sln | 27 +++ .../slice_hash_table_test.vcxproj | 199 +++++++++++++++++ .../slice_hash_table_test.vcxproj.filters | 21 ++ 12 files changed, 616 insertions(+), 32 deletions(-) create mode 100644 test/core/slice/slice_hash_table_test.c create mode 100644 vsprojects/vcxproj/test/slice_hash_table_test/slice_hash_table_test.vcxproj create mode 100644 vsprojects/vcxproj/test/slice_hash_table_test/slice_hash_table_test.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index a6a2af65d5d..230a3d1ac0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -493,6 +493,7 @@ add_dependencies(buildtests_c sequential_connectivity_test) add_dependencies(buildtests_c server_chttp2_test) add_dependencies(buildtests_c server_test) add_dependencies(buildtests_c slice_buffer_test) +add_dependencies(buildtests_c slice_hash_table_test) add_dependencies(buildtests_c slice_string_helpers_test) add_dependencies(buildtests_c slice_test) add_dependencies(buildtests_c sockaddr_resolver_test) @@ -8007,6 +8008,37 @@ target_link_libraries(slice_buffer_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +add_executable(slice_hash_table_test + test/core/slice/slice_hash_table_test.c +) + + +target_include_directories(slice_hash_table_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_BUILD_INCLUDE_DIR} + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include +) + +target_link_libraries(slice_hash_table_test + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(slice_string_helpers_test test/core/slice/slice_string_helpers_test.c ) diff --git a/Makefile b/Makefile index 8898939af9e..a233f712e48 100644 --- a/Makefile +++ b/Makefile @@ -1073,6 +1073,7 @@ server_chttp2_test: $(BINDIR)/$(CONFIG)/server_chttp2_test server_fuzzer: $(BINDIR)/$(CONFIG)/server_fuzzer server_test: $(BINDIR)/$(CONFIG)/server_test slice_buffer_test: $(BINDIR)/$(CONFIG)/slice_buffer_test +slice_hash_table_test: $(BINDIR)/$(CONFIG)/slice_hash_table_test slice_string_helpers_test: $(BINDIR)/$(CONFIG)/slice_string_helpers_test slice_test: $(BINDIR)/$(CONFIG)/slice_test sockaddr_resolver_test: $(BINDIR)/$(CONFIG)/sockaddr_resolver_test @@ -1443,6 +1444,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/server_chttp2_test \ $(BINDIR)/$(CONFIG)/server_test \ $(BINDIR)/$(CONFIG)/slice_buffer_test \ + $(BINDIR)/$(CONFIG)/slice_hash_table_test \ $(BINDIR)/$(CONFIG)/slice_string_helpers_test \ $(BINDIR)/$(CONFIG)/slice_test \ $(BINDIR)/$(CONFIG)/sockaddr_resolver_test \ @@ -1915,6 +1917,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/server_test || ( echo test server_test failed ; exit 1 ) $(E) "[RUN] Testing slice_buffer_test" $(Q) $(BINDIR)/$(CONFIG)/slice_buffer_test || ( echo test slice_buffer_test failed ; exit 1 ) + $(E) "[RUN] Testing slice_hash_table_test" + $(Q) $(BINDIR)/$(CONFIG)/slice_hash_table_test || ( echo test slice_hash_table_test failed ; exit 1 ) $(E) "[RUN] Testing slice_string_helpers_test" $(Q) $(BINDIR)/$(CONFIG)/slice_string_helpers_test || ( echo test slice_string_helpers_test failed ; exit 1 ) $(E) "[RUN] Testing slice_test" @@ -12349,6 +12353,38 @@ endif endif +SLICE_HASH_TABLE_TEST_SRC = \ + test/core/slice/slice_hash_table_test.c \ + +SLICE_HASH_TABLE_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SLICE_HASH_TABLE_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/slice_hash_table_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/slice_hash_table_test: $(SLICE_HASH_TABLE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(SLICE_HASH_TABLE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/slice_hash_table_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/slice/slice_hash_table_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_slice_hash_table_test: $(SLICE_HASH_TABLE_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(SLICE_HASH_TABLE_TEST_OBJS:.o=.dep) +endif +endif + + SLICE_STRING_HELPERS_TEST_SRC = \ test/core/slice/slice_string_helpers_test.c \ diff --git a/build.yaml b/build.yaml index f4461e40ef1..b40834e16d4 100644 --- a/build.yaml +++ b/build.yaml @@ -2807,6 +2807,16 @@ targets: - grpc - gpr_test_util - gpr +- name: slice_hash_table_test + build: test + language: c + src: + - test/core/slice/slice_hash_table_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: slice_string_helpers_test build: test language: c diff --git a/src/core/lib/slice/slice_hash_table.c b/src/core/lib/slice/slice_hash_table.c index 7a8b79cc336..a33f6069fd5 100644 --- a/src/core/lib/slice/slice_hash_table.c +++ b/src/core/lib/slice/slice_hash_table.c @@ -44,6 +44,7 @@ struct grpc_slice_hash_table { gpr_refcount refs; void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value); size_t size; + size_t max_num_probes; grpc_slice_hash_table_entry* entries; }; @@ -51,32 +52,22 @@ static bool is_empty(grpc_slice_hash_table_entry* entry) { return entry->value == NULL; } -// Helper function for insert and get operations that performs quadratic -// probing (https://en.wikipedia.org/wiki/Quadratic_probing). -static size_t grpc_slice_hash_table_find_index( - const grpc_slice_hash_table* table, const grpc_slice key, bool find_empty) { - size_t hash = grpc_slice_hash(key); - for (size_t i = 0; i < table->size; ++i) { - const size_t idx = (hash + i * i) % table->size; - if (is_empty(&table->entries[idx])) { - return find_empty ? idx : table->size; - } - if (grpc_slice_eq(table->entries[idx].key, key)) { - return idx; - } - } - return table->size; // Not found. -} - static void grpc_slice_hash_table_add(grpc_slice_hash_table* table, grpc_slice key, void* value) { GPR_ASSERT(value != NULL); - const size_t idx = - grpc_slice_hash_table_find_index(table, key, true /* find_empty */); - GPR_ASSERT(idx != table->size); // Table should never be full. - grpc_slice_hash_table_entry* entry = &table->entries[idx]; - entry->key = key; - entry->value = value; + const size_t hash = grpc_slice_hash(key); + for (size_t offset = 0; offset < table->size; ++offset) { + const size_t idx = (hash + offset) % table->size; + if (is_empty(&table->entries[idx])) { + table->entries[idx].key = key; + table->entries[idx].value = value; + // Keep track of the maximum number of probes needed, since this + // provides an upper bound for lookups. + if (offset > table->max_num_probes) table->max_num_probes = offset; + return; + } + } + GPR_ASSERT(false); // Table should never be full. } grpc_slice_hash_table* grpc_slice_hash_table_create( @@ -85,8 +76,7 @@ grpc_slice_hash_table* grpc_slice_hash_table_create( grpc_slice_hash_table* table = gpr_zalloc(sizeof(*table)); gpr_ref_init(&table->refs, 1); table->destroy_value = destroy_value; - // Quadratic probing gets best performance when the table is no more - // than half full. + // Keep load factor low to improve performance of lookups. table->size = num_entries * 2; const size_t entry_size = sizeof(grpc_slice_hash_table_entry) * table->size; table->entries = gpr_zalloc(entry_size); @@ -119,8 +109,15 @@ void grpc_slice_hash_table_unref(grpc_exec_ctx* exec_ctx, void* grpc_slice_hash_table_get(const grpc_slice_hash_table* table, const grpc_slice key) { - const size_t idx = - grpc_slice_hash_table_find_index(table, key, false /* find_empty */); - if (idx == table->size) return NULL; // Not found. - return table->entries[idx].value; + const size_t hash = grpc_slice_hash(key); + // We cap the number of probes at the max number recorded when + // populating the table. + for (size_t offset = 0; offset <= table->max_num_probes; ++offset) { + const size_t idx = (hash + offset) % table->size; + if (is_empty(&table->entries[idx])) break; + if (grpc_slice_eq(table->entries[idx].key, key)) { + return table->entries[idx].value; + } + } + return NULL; // Not found. } diff --git a/src/core/lib/slice/slice_hash_table.h b/src/core/lib/slice/slice_hash_table.h index 81e92b6e21f..1e61c5eb116 100644 --- a/src/core/lib/slice/slice_hash_table.h +++ b/src/core/lib/slice/slice_hash_table.h @@ -37,8 +37,8 @@ /** Hash table implementation. * * This implementation uses open addressing - * (https://en.wikipedia.org/wiki/Open_addressing) with quadratic - * probing (https://en.wikipedia.org/wiki/Quadratic_probing). + * (https://en.wikipedia.org/wiki/Open_addressing) with linear + * probing (https://en.wikipedia.org/wiki/Linear_probing). * * The keys are \a grpc_slice objects. The values are arbitrary pointers * with a common destroy function. diff --git a/test/core/slice/BUILD b/test/core/slice/BUILD index 4d64d0a8183..18cf6f60af0 100644 --- a/test/core/slice/BUILD +++ b/test/core/slice/BUILD @@ -47,12 +47,33 @@ cc_test( ) cc_test( - name = "slice_buffer_test", + name = "slice_test", + srcs = ["slice_test.c"], + deps = ["//:grpc", "//test/core/util:grpc_test_util", "//:gpr", "//test/core/util:gpr_test_util"], + copts = ['-std=c99'] +) + +cc_test( + name = "slice_string_helpers_test", srcs = ["slice_string_helpers_test.c"], deps = ["//:grpc", "//test/core/util:grpc_test_util", "//:gpr", "//test/core/util:gpr_test_util"], copts = ['-std=c99'] ) +cc_test( + name = "slice_buffer_test", + srcs = ["slice_buffer_test.c"], + deps = ["//:grpc", "//test/core/util:grpc_test_util", "//:gpr", "//test/core/util:gpr_test_util"], + copts = ['-std=c99'] +) + +cc_test( + name = "slice_hash_table_test", + srcs = ["slice_hash_table_test.c"], + deps = ["//:grpc", "//test/core/util:grpc_test_util", "//:gpr", "//test/core/util:gpr_test_util"], + copts = ['-std=c99'] +) + cc_test( name = "b64_test", srcs = ["b64_test.c"], diff --git a/test/core/slice/slice_hash_table_test.c b/test/core/slice/slice_hash_table_test.c new file mode 100644 index 00000000000..24a4354b8b6 --- /dev/null +++ b/test/core/slice/slice_hash_table_test.c @@ -0,0 +1,202 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/slice/slice_hash_table.h" + +#include + +#include +#include +#include + +#include "src/core/lib/slice/slice_internal.h" +#include "test/core/util/test_config.h" + +typedef struct { + char* key; + char* value; +} test_entry; + +static void populate_entries(const test_entry* input, size_t num_entries, + grpc_slice_hash_table_entry* output) { + for (size_t i = 0; i < num_entries; ++i) { + output[i].key = grpc_slice_from_copied_string(gpr_strdup(input[i].key)); + output[i].value = gpr_strdup(input[i].value); + } +} + +static void check_values(const test_entry* input, size_t num_entries, + grpc_slice_hash_table* table) { + for (size_t i = 0; i < num_entries; ++i) { + grpc_slice key = grpc_slice_from_static_string(input[i].key); + char* actual = grpc_slice_hash_table_get(table, key); + GPR_ASSERT(actual != NULL); + GPR_ASSERT(strcmp(actual, input[i].value) == 0); + grpc_slice_unref(key); + } +} + +static void check_non_existent_value(const char* key_string, + grpc_slice_hash_table* table) { + grpc_slice key = grpc_slice_from_static_string(key_string); + GPR_ASSERT(grpc_slice_hash_table_get(table, key) == NULL); + grpc_slice_unref(key); +} + +static void destroy_string(grpc_exec_ctx* exec_ctx, void* value) { + gpr_free(value); +} + +static void test_slice_hash_table() { + const test_entry test_entries[] = { + {"key_0", "value_0"}, + {"key_1", "value_1"}, + {"key_2", "value_2"}, + {"key_3", "value_3"}, + {"key_4", "value_4"}, + {"key_5", "value_5"}, + {"key_6", "value_6"}, + {"key_7", "value_7"}, + {"key_8", "value_8"}, + {"key_9", "value_9"}, + {"key_10", "value_10"}, + {"key_11", "value_11"}, + {"key_12", "value_12"}, + {"key_13", "value_13"}, + {"key_14", "value_14"}, + {"key_15", "value_15"}, + {"key_16", "value_16"}, + {"key_17", "value_17"}, + {"key_18", "value_18"}, + {"key_19", "value_19"}, + {"key_20", "value_20"}, + {"key_21", "value_21"}, + {"key_22", "value_22"}, + {"key_23", "value_23"}, + {"key_24", "value_24"}, + {"key_25", "value_25"}, + {"key_26", "value_26"}, + {"key_27", "value_27"}, + {"key_28", "value_28"}, + {"key_29", "value_29"}, + {"key_30", "value_30"}, + {"key_31", "value_31"}, + {"key_32", "value_32"}, + {"key_33", "value_33"}, + {"key_34", "value_34"}, + {"key_35", "value_35"}, + {"key_36", "value_36"}, + {"key_37", "value_37"}, + {"key_38", "value_38"}, + {"key_39", "value_39"}, + {"key_40", "value_40"}, + {"key_41", "value_41"}, + {"key_42", "value_42"}, + {"key_43", "value_43"}, + {"key_44", "value_44"}, + {"key_45", "value_45"}, + {"key_46", "value_46"}, + {"key_47", "value_47"}, + {"key_48", "value_48"}, + {"key_49", "value_49"}, + {"key_50", "value_50"}, + {"key_51", "value_51"}, + {"key_52", "value_52"}, + {"key_53", "value_53"}, + {"key_54", "value_54"}, + {"key_55", "value_55"}, + {"key_56", "value_56"}, + {"key_57", "value_57"}, + {"key_58", "value_58"}, + {"key_59", "value_59"}, + {"key_60", "value_60"}, + {"key_61", "value_61"}, + {"key_62", "value_62"}, + {"key_63", "value_63"}, + {"key_64", "value_64"}, + {"key_65", "value_65"}, + {"key_66", "value_66"}, + {"key_67", "value_67"}, + {"key_68", "value_68"}, + {"key_69", "value_69"}, + {"key_70", "value_70"}, + {"key_71", "value_71"}, + {"key_72", "value_72"}, + {"key_73", "value_73"}, + {"key_74", "value_74"}, + {"key_75", "value_75"}, + {"key_76", "value_76"}, + {"key_77", "value_77"}, + {"key_78", "value_78"}, + {"key_79", "value_79"}, + {"key_80", "value_80"}, + {"key_81", "value_81"}, + {"key_82", "value_82"}, + {"key_83", "value_83"}, + {"key_84", "value_84"}, + {"key_85", "value_85"}, + {"key_86", "value_86"}, + {"key_87", "value_87"}, + {"key_88", "value_88"}, + {"key_89", "value_89"}, + {"key_90", "value_90"}, + {"key_91", "value_91"}, + {"key_92", "value_92"}, + {"key_93", "value_93"}, + {"key_94", "value_94"}, + {"key_95", "value_95"}, + {"key_96", "value_96"}, + {"key_97", "value_97"}, + {"key_98", "value_98"}, + {"key_99", "value_99"}, + }; + const size_t num_entries = GPR_ARRAY_SIZE(test_entries); + // Construct table. + grpc_slice_hash_table_entry entries[num_entries]; + populate_entries(test_entries, num_entries, entries); + grpc_slice_hash_table* table = + grpc_slice_hash_table_create(num_entries, entries, destroy_string); + // Check contents of table. + check_values(test_entries, num_entries, table); + check_non_existent_value("XX", table); + // Clean up. + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_slice_hash_table_unref(&exec_ctx, table); + grpc_exec_ctx_finish(&exec_ctx); +} + +int main(int argc, char **argv) { + grpc_test_init(argc, argv); + test_slice_hash_table(); + return 0; +} diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index aa4869e1c8a..9635dbec839 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -1978,6 +1978,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "slice_hash_table_test", + "src": [ + "test/core/slice/slice_hash_table_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 6338ea70127..eb8fb20d7de 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2071,6 +2071,28 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "slice_hash_table_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 6ae8bfa74de..539f474f9ec 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -1440,6 +1440,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "slice_buffer_test", "vcxpro {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "slice_hash_table_test", "vcxproj\test\slice_hash_table_test\slice_hash_table_test.vcxproj", "{B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "slice_string_helpers_test", "vcxproj\test\slice_string_helpers_test\slice_string_helpers_test.vcxproj", "{419167BB-C3F5-DDEA-403A-394D1902DE65}" ProjectSection(myProperties) = preProject lib = "False" @@ -3823,6 +3834,22 @@ Global {F0FA4A41-5695-580A-DCDA-EC719CB041B0}.Release-DLL|Win32.Build.0 = Release|Win32 {F0FA4A41-5695-580A-DCDA-EC719CB041B0}.Release-DLL|x64.ActiveCfg = Release|x64 {F0FA4A41-5695-580A-DCDA-EC719CB041B0}.Release-DLL|x64.Build.0 = Release|x64 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Debug|Win32.ActiveCfg = Debug|Win32 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Debug|x64.ActiveCfg = Debug|x64 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Release|Win32.ActiveCfg = Release|Win32 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Release|x64.ActiveCfg = Release|x64 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Debug|Win32.Build.0 = Debug|Win32 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Debug|x64.Build.0 = Debug|x64 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Release|Win32.Build.0 = Release|Win32 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Release|x64.Build.0 = Release|x64 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Debug-DLL|x64.Build.0 = Debug|x64 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Release-DLL|Win32.Build.0 = Release|Win32 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Release-DLL|x64.ActiveCfg = Release|x64 + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9}.Release-DLL|x64.Build.0 = Release|x64 {419167BB-C3F5-DDEA-403A-394D1902DE65}.Debug|Win32.ActiveCfg = Debug|Win32 {419167BB-C3F5-DDEA-403A-394D1902DE65}.Debug|x64.ActiveCfg = Debug|x64 {419167BB-C3F5-DDEA-403A-394D1902DE65}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/test/slice_hash_table_test/slice_hash_table_test.vcxproj b/vsprojects/vcxproj/test/slice_hash_table_test/slice_hash_table_test.vcxproj new file mode 100644 index 00000000000..0fccfdcaa56 --- /dev/null +++ b/vsprojects/vcxproj/test/slice_hash_table_test/slice_hash_table_test.vcxproj @@ -0,0 +1,199 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {B3BC3481-FCD3-BA52-C975-E65CDB1CE9A9} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + slice_hash_table_test + static + Debug + static + Debug + + + slice_hash_table_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/slice_hash_table_test/slice_hash_table_test.vcxproj.filters b/vsprojects/vcxproj/test/slice_hash_table_test/slice_hash_table_test.vcxproj.filters new file mode 100644 index 00000000000..19738526781 --- /dev/null +++ b/vsprojects/vcxproj/test/slice_hash_table_test/slice_hash_table_test.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\core\slice + + + + + + {e11f5007-84da-0229-9118-2a2bb17f956c} + + + {a11f4770-5e13-eb1a-a848-8667dde98812} + + + {fb1262a8-0a70-bc85-579a-6ebfffbf25b5} + + + + From e0a873784adaff3a0dfd06d16cb621d3eddf3443 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 19 Apr 2017 13:39:29 -0700 Subject: [PATCH 20/51] Fix bug in service config code. --- src/core/lib/transport/service_config.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/lib/transport/service_config.c b/src/core/lib/transport/service_config.c index 3cda394723d..6aecb7fa936 100644 --- a/src/core/lib/transport/service_config.c +++ b/src/core/lib/transport/service_config.c @@ -162,7 +162,6 @@ static char* parse_json_method_name(grpc_json* json) { static bool parse_json_method_config( grpc_exec_ctx* exec_ctx, grpc_json* json, void* (*create_value)(const grpc_json* method_config_json), - void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value), grpc_slice_hash_table_entry* entries, size_t* idx) { // Construct value. void* method_config = create_value(json); @@ -190,7 +189,6 @@ static bool parse_json_method_config( } success = true; done: - destroy_value(exec_ctx, method_config); gpr_strvec_destroy(&paths); return success; } @@ -219,8 +217,8 @@ grpc_slice_hash_table* grpc_service_config_create_method_config_table( size_t idx = 0; for (grpc_json* method = field->child; method != NULL; method = method->next) { - if (!parse_json_method_config(exec_ctx, method, create_value, - destroy_value, entries, &idx)) { + if (!parse_json_method_config(exec_ctx, method, create_value, entries, + &idx)) { return NULL; } } From 880530694192f9be13b8767bfab7a06fdda9697e Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 19 Apr 2017 13:50:06 -0700 Subject: [PATCH 21/51] Revert API to change default queue for a stub --- src/compiler/objective_c_generator.cc | 34 ++++---------- src/objective-c/tests/InteropTests.m | 68 +++++---------------------- 2 files changed, 21 insertions(+), 81 deletions(-) diff --git a/src/compiler/objective_c_generator.cc b/src/compiler/objective_c_generator.cc index ba89b28b7b5..1d7faf120dc 100644 --- a/src/compiler/objective_c_generator.cc +++ b/src/compiler/objective_c_generator.cc @@ -149,20 +149,17 @@ void PrintMethodDeclarations(Printer *printer, const MethodDescriptor *method) { void PrintSimpleImplementation(Printer *printer, const MethodDescriptor *method, map< ::grpc::string, ::grpc::string> vars) { printer->Print("{\n"); - printer->Print(vars, " GRPCProtoCall *rpc = [self RPCTo$method_name$With"); + printer->Print(vars, " [[self RPCTo$method_name$With"); if (method->client_streaming()) { printer->Print("RequestsWriter:requestWriter"); } else { printer->Print("Request:request"); } if (method->server_streaming()) { - printer->Print(" eventHandler:eventHandler];\n"); + printer->Print(" eventHandler:eventHandler] start];\n"); } else { - printer->Print(" handler:handler];\n"); + printer->Print(" handler:handler] start];\n"); } - printer->Print( - " [rpc setResponseDispatchQueue:_defaultResponseDispatchQueue];\n"); - printer->Print(" [rpc start];\n"); printer->Print("}\n"); } @@ -170,29 +167,23 @@ void PrintAdvancedImplementation(Printer *printer, const MethodDescriptor *method, map< ::grpc::string, ::grpc::string> vars) { printer->Print("{\n"); - printer->Print( - vars, " GRPCProtoCall *rpc = [self RPCToMethod:@\"$method_name$\"\n"); + printer->Print(vars, " return [self RPCToMethod:@\"$method_name$\"\n"); - printer->Print(" requestsWriter:"); + printer->Print(" requestsWriter:"); if (method->client_streaming()) { printer->Print("requestWriter\n"); } else { printer->Print("[GRXWriter writerWithValue:request]\n"); } - printer->Print( - vars, - " responseClass:[$response_class$ class]\n"); + printer->Print(vars, " responseClass:[$response_class$ class]\n"); - printer->Print(" responsesWriteable:[GRXWriteable "); + printer->Print(" responsesWriteable:[GRXWriteable "); if (method->server_streaming()) { printer->Print("writeableWithEventHandler:eventHandler]];\n"); } else { printer->Print("writeableWithSingleHandler:handler]];\n"); } - printer->Print( - " [rpc setResponseDispatchQueue:_defaultResponseDispatchQueue];\n"); - printer->Print(" return rpc;\n"); printer->Print("}\n"); } @@ -243,8 +234,6 @@ void PrintMethodImplementations(Printer *printer, "- (instancetype)initWithHost:(NSString *)host" " NS_DESIGNATED_INITIALIZER;\n"); printer.Print("+ (instancetype)serviceWithHost:(NSString *)host;\n"); - printer.Print( - "- (void)setDefaultResponseDispatchQueue:(dispatch_queue_t)queue;\n"); printer.Print("@end\n"); } return output; @@ -262,15 +251,12 @@ void PrintMethodImplementations(Printer *printer, {"service_class", ServiceClassName(service)}, {"package", service->file()->package()}}; - printer.Print(vars, "@implementation $service_class$ {\n"); - printer.Print(vars, " dispatch_queue_t _defaultResponseDispatchQueue;\n"); - printer.Print(vars, "}\n\n"); + printer.Print(vars, "@implementation $service_class$\n\n"); printer.Print("// Designated initializer\n"); printer.Print("- (instancetype)initWithHost:(NSString *)host {\n"); printer.Print( vars, - " _defaultResponseDispatchQueue = dispatch_get_main_queue();\n" " return (self = [super initWithHost:host" " packageName:@\"$package$\" serviceName:@\"$service_name$\"]);\n"); printer.Print("}\n\n"); @@ -284,10 +270,6 @@ void PrintMethodImplementations(Printer *printer, printer.Print("}\n\n"); printer.Print("+ (instancetype)serviceWithHost:(NSString *)host {\n"); printer.Print(" return [[self alloc] initWithHost:host];\n"); - printer.Print("}\n\n"); - printer.Print( - "- (void)setDefaultResponseDispatchQueue:(dispatch_queue_t)queue {\n"); - printer.Print(" _defaultResponseDispatchQueue = queue;\n"); printer.Print("}\n\n\n"); for (int i = 0; i < service->method_count(); i++) { diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index 4466d6810cc..e7d28f4e459 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -459,23 +459,17 @@ - (void)testAlternateDispatchQueue { XCTAssertNotNil(self.class.host); - __weak XCTestExpectation *expectation1 = [self expectationWithDescription:@"AlternateDispatchQueue1"]; + NSNumber *kPayloadSize = @256; + id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:kPayloadSize + requestedResponseSize:kPayloadSize]; - NSArray *requests = @[@27182, @8, @1828, @45904]; - NSArray *responses = @[@31415, @9, @2653, @58979]; + __weak XCTestExpectation *expectation1 = [self expectationWithDescription:@"AlternateDispatchQueue1"]; // Set the default dispatch queue - NSString *queue1_label = @"test.queue1"; - NSString *queue2_label = @"test.queue2"; - dispatch_queue_t queue1 = dispatch_queue_create([queue1_label UTF8String], DISPATCH_QUEUE_SERIAL); - dispatch_queue_t queue2 = dispatch_queue_create([queue2_label UTF8String], DISPATCH_QUEUE_SERIAL); - [_service setDefaultResponseDispatchQueue:queue1]; + NSString *queue_label = @"test.queue1"; + dispatch_queue_t queue = dispatch_queue_create([queue_label UTF8String], DISPATCH_QUEUE_SERIAL); GRXBufferedPipe *requestsBuffer1 = [[GRXBufferedPipe alloc] init]; - __block int index = 0; - - id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:requests[index] - requestedResponseSize:responses[index]]; [requestsBuffer1 writeValue:request]; [_service fullDuplexCallWithRequestsWriter:requestsBuffer1 @@ -485,26 +479,11 @@ XCTAssertNil(error, @"Finished with unexpected error: %@", error); XCTAssertTrue(done || response, @"Event handler called without an event."); NSString *label = [NSString stringWithUTF8String:dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL)]; - XCTAssert([label isEqualToString:queue1_label]); - - if (response) { - XCTAssertLessThan(index, 4, @"More than 4 responses received."); - id expected = [RMTStreamingOutputCallResponse messageWithPayloadSize:responses[index]]; - XCTAssertEqualObjects(response, expected); - index += 1; - if (index < 4) { - id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:requests[index] - requestedResponseSize:responses[index]]; - [requestsBuffer1 writeValue:request]; - } else { - [requestsBuffer1 writesFinishedWithError:nil]; - } - } + NSLog(@"main queue label:%@", label); + NSString *main_queue_label = [NSString stringWithUTF8String:dispatch_queue_get_label(dispatch_get_main_queue())]; + XCTAssert([label isEqualToString:main_queue_label]); - if (done) { - XCTAssertEqual(index, 4, @"Received %i responses instead of 4.", index); - [expectation1 fulfill]; - } + [expectation1 fulfill]; }]; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; @@ -513,8 +492,6 @@ __weak XCTestExpectation *expectation2 = [self expectationWithDescription:@"AlternateDispatchQueue2"]; GRXBufferedPipe *requestsBuffer2 = [[GRXBufferedPipe alloc] init]; - index = 0; - [requestsBuffer2 writeValue:request]; GRPCProtoCall *rpc = [_service RPCToFullDuplexCallWithRequestsWriter:requestsBuffer2 @@ -524,32 +501,13 @@ XCTAssertNil(error, @"Finished with unexpected error: %@", error); XCTAssertTrue(done || response, @"Event handler called without an event."); NSString *label = [NSString stringWithUTF8String:dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL)]; - XCTAssert([label isEqualToString:queue2_label]); - - if (response) { - XCTAssertLessThan(index, 4, @"More than 4 responses received."); - id expected = [RMTStreamingOutputCallResponse messageWithPayloadSize:responses[index]]; - XCTAssertEqualObjects(response, expected); - index += 1; - if (index < 4) { - id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:requests[index] - requestedResponseSize:responses[index]]; - [requestsBuffer2 writeValue:request]; - } else { - [requestsBuffer2 writesFinishedWithError:nil]; - } - } - - if (done) { - XCTAssertEqual(index, 4, @"Received %i responses instead of 4.", index); - [expectation2 fulfill]; - } + XCTAssert([label isEqualToString:queue_label]); + [expectation2 fulfill]; }]; - [rpc setResponseDispatchQueue:queue2]; + [rpc setResponseDispatchQueue:queue]; [rpc start]; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; - [_service setDefaultResponseDispatchQueue:dispatch_get_main_queue()]; } @end From b965ff6bf764c9144862af6e4fb25bd90ecd6f55 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 19 Apr 2017 14:27:47 -0700 Subject: [PATCH 22/51] Move test to GRPCClientTests --- src/objective-c/tests/GRPCClientTests.m | 55 +++++++++++++++++++++++++ src/objective-c/tests/InteropTests.m | 53 ------------------------ 2 files changed, 55 insertions(+), 53 deletions(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 76c15003f60..0a631d182ab 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -353,4 +353,59 @@ static GRPCProtoMethod *kUnaryCallMethod; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +- (void)testAlternateDispatchQueue { + const int32_t kPayloadSize = 100; + RMTSimpleRequest *request = [RMTSimpleRequest message]; + request.responseSize = kPayloadSize; + + __weak XCTestExpectation *expectation1 = [self expectationWithDescription:@"AlternateDispatchQueue1"]; + + // Use default (main) dispatch queue + NSString *main_queue_label = [NSString stringWithUTF8String:dispatch_queue_get_label(dispatch_get_main_queue())]; + + GRXWriter *requestsWriter1 = [GRXWriter writerWithValue:[request data]]; + + GRPCCall *call1 = [[GRPCCall alloc] initWithHost:kHostAddress + path:kUnaryCallMethod.HTTPPath + requestsWriter:requestsWriter1]; + + id responsesWriteable1 = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + NSString *label = [NSString stringWithUTF8String:dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL)]; + XCTAssert([label isEqualToString:main_queue_label]); + + [expectation1 fulfill]; + } completionHandler:^(NSError *errorOrNil) { + }]; + + [call1 startWithWriteable:responsesWriteable1]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; + + // Use a custom queue + __weak XCTestExpectation *expectation2 = [self expectationWithDescription:@"AlternateDispatchQueue2"]; + + NSString *queue_label = @"test.queue1"; + dispatch_queue_t queue = dispatch_queue_create([queue_label UTF8String], DISPATCH_QUEUE_SERIAL); + + GRXWriter *requestsWriter2 = [GRXWriter writerWithValue:[request data]]; + + GRPCCall *call2 = [[GRPCCall alloc] initWithHost:kHostAddress + path:kUnaryCallMethod.HTTPPath + requestsWriter:requestsWriter2]; + + [call2 setResponseDispatchQueue:queue]; + + id responsesWriteable2 = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + NSString *label = [NSString stringWithUTF8String:dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL)]; + XCTAssert([label isEqualToString:queue_label]); + + [expectation2 fulfill]; + } completionHandler:^(NSError *errorOrNil) { + }]; + + [call2 startWithWriteable:responsesWriteable2]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; +} + @end diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index e7d28f4e459..91053568690 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -457,57 +457,4 @@ [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } -- (void)testAlternateDispatchQueue { - XCTAssertNotNil(self.class.host); - NSNumber *kPayloadSize = @256; - id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:kPayloadSize - requestedResponseSize:kPayloadSize]; - - __weak XCTestExpectation *expectation1 = [self expectationWithDescription:@"AlternateDispatchQueue1"]; - - // Set the default dispatch queue - NSString *queue_label = @"test.queue1"; - dispatch_queue_t queue = dispatch_queue_create([queue_label UTF8String], DISPATCH_QUEUE_SERIAL); - GRXBufferedPipe *requestsBuffer1 = [[GRXBufferedPipe alloc] init]; - - [requestsBuffer1 writeValue:request]; - - [_service fullDuplexCallWithRequestsWriter:requestsBuffer1 - eventHandler:^(BOOL done, - RMTStreamingOutputCallResponse *response, - NSError *error) { - XCTAssertNil(error, @"Finished with unexpected error: %@", error); - XCTAssertTrue(done || response, @"Event handler called without an event."); - NSString *label = [NSString stringWithUTF8String:dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL)]; - NSLog(@"main queue label:%@", label); - NSString *main_queue_label = [NSString stringWithUTF8String:dispatch_queue_get_label(dispatch_get_main_queue())]; - XCTAssert([label isEqualToString:main_queue_label]); - - [expectation1 fulfill]; - }]; - - [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; - - // Test overriding default queue with another queue - __weak XCTestExpectation *expectation2 = [self expectationWithDescription:@"AlternateDispatchQueue2"]; - GRXBufferedPipe *requestsBuffer2 = [[GRXBufferedPipe alloc] init]; - - [requestsBuffer2 writeValue:request]; - - GRPCProtoCall *rpc = [_service RPCToFullDuplexCallWithRequestsWriter:requestsBuffer2 - eventHandler:^(BOOL done, - RMTStreamingOutputCallResponse *response, - NSError *error) { - XCTAssertNil(error, @"Finished with unexpected error: %@", error); - XCTAssertTrue(done || response, @"Event handler called without an event."); - NSString *label = [NSString stringWithUTF8String:dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL)]; - XCTAssert([label isEqualToString:queue_label]); - [expectation2 fulfill]; - }]; - [rpc setResponseDispatchQueue:queue]; - [rpc start]; - - [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; -} - @end From a2529ce53ae729b1ebcc0fa241ec9671c891dad1 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 19 Apr 2017 14:32:47 -0700 Subject: [PATCH 23/51] Minor comment polish --- src/objective-c/GRPCClient/GRPCCall.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/GRPCClient/GRPCCall.h b/src/objective-c/GRPCClient/GRPCCall.h index 85c5b47c8ea..5e9324c4456 100644 --- a/src/objective-c/GRPCClient/GRPCCall.h +++ b/src/objective-c/GRPCClient/GRPCCall.h @@ -254,7 +254,7 @@ extern id const kGRPCTrailersKey; + (void)setCallSafety:(GRPCCallSafety)callSafety host:(NSString *)host path:(NSString *)path; /** - * Set the dispatch queue to be used for queue responses. + * Set the dispatch queue to be used for callbacks. * * This configuration is only effective before the call starts. */ From ba390352c9d232d5e03e5821c06aecd2fd1f98d2 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 20 Apr 2017 06:59:26 -0700 Subject: [PATCH 24/51] clang-format --- src/core/lib/slice/slice_hash_table.c | 4 +- test/core/slice/slice_hash_table_test.c | 136 ++++++------------------ 2 files changed, 37 insertions(+), 103 deletions(-) diff --git a/src/core/lib/slice/slice_hash_table.c b/src/core/lib/slice/slice_hash_table.c index a33f6069fd5..444f22aa196 100644 --- a/src/core/lib/slice/slice_hash_table.c +++ b/src/core/lib/slice/slice_hash_table.c @@ -42,7 +42,7 @@ struct grpc_slice_hash_table { gpr_refcount refs; - void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value); + void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value); size_t size; size_t max_num_probes; grpc_slice_hash_table_entry* entries; @@ -72,7 +72,7 @@ static void grpc_slice_hash_table_add(grpc_slice_hash_table* table, grpc_slice_hash_table* grpc_slice_hash_table_create( size_t num_entries, grpc_slice_hash_table_entry* entries, - void (*destroy_value)(grpc_exec_ctx *exec_ctx, void *value)) { + void (*destroy_value)(grpc_exec_ctx* exec_ctx, void* value)) { grpc_slice_hash_table* table = gpr_zalloc(sizeof(*table)); gpr_ref_init(&table->refs, 1); table->destroy_value = destroy_value; diff --git a/test/core/slice/slice_hash_table_test.c b/test/core/slice/slice_hash_table_test.c index 24a4354b8b6..9eba57e70aa 100644 --- a/test/core/slice/slice_hash_table_test.c +++ b/test/core/slice/slice_hash_table_test.c @@ -79,106 +79,40 @@ static void destroy_string(grpc_exec_ctx* exec_ctx, void* value) { static void test_slice_hash_table() { const test_entry test_entries[] = { - {"key_0", "value_0"}, - {"key_1", "value_1"}, - {"key_2", "value_2"}, - {"key_3", "value_3"}, - {"key_4", "value_4"}, - {"key_5", "value_5"}, - {"key_6", "value_6"}, - {"key_7", "value_7"}, - {"key_8", "value_8"}, - {"key_9", "value_9"}, - {"key_10", "value_10"}, - {"key_11", "value_11"}, - {"key_12", "value_12"}, - {"key_13", "value_13"}, - {"key_14", "value_14"}, - {"key_15", "value_15"}, - {"key_16", "value_16"}, - {"key_17", "value_17"}, - {"key_18", "value_18"}, - {"key_19", "value_19"}, - {"key_20", "value_20"}, - {"key_21", "value_21"}, - {"key_22", "value_22"}, - {"key_23", "value_23"}, - {"key_24", "value_24"}, - {"key_25", "value_25"}, - {"key_26", "value_26"}, - {"key_27", "value_27"}, - {"key_28", "value_28"}, - {"key_29", "value_29"}, - {"key_30", "value_30"}, - {"key_31", "value_31"}, - {"key_32", "value_32"}, - {"key_33", "value_33"}, - {"key_34", "value_34"}, - {"key_35", "value_35"}, - {"key_36", "value_36"}, - {"key_37", "value_37"}, - {"key_38", "value_38"}, - {"key_39", "value_39"}, - {"key_40", "value_40"}, - {"key_41", "value_41"}, - {"key_42", "value_42"}, - {"key_43", "value_43"}, - {"key_44", "value_44"}, - {"key_45", "value_45"}, - {"key_46", "value_46"}, - {"key_47", "value_47"}, - {"key_48", "value_48"}, - {"key_49", "value_49"}, - {"key_50", "value_50"}, - {"key_51", "value_51"}, - {"key_52", "value_52"}, - {"key_53", "value_53"}, - {"key_54", "value_54"}, - {"key_55", "value_55"}, - {"key_56", "value_56"}, - {"key_57", "value_57"}, - {"key_58", "value_58"}, - {"key_59", "value_59"}, - {"key_60", "value_60"}, - {"key_61", "value_61"}, - {"key_62", "value_62"}, - {"key_63", "value_63"}, - {"key_64", "value_64"}, - {"key_65", "value_65"}, - {"key_66", "value_66"}, - {"key_67", "value_67"}, - {"key_68", "value_68"}, - {"key_69", "value_69"}, - {"key_70", "value_70"}, - {"key_71", "value_71"}, - {"key_72", "value_72"}, - {"key_73", "value_73"}, - {"key_74", "value_74"}, - {"key_75", "value_75"}, - {"key_76", "value_76"}, - {"key_77", "value_77"}, - {"key_78", "value_78"}, - {"key_79", "value_79"}, - {"key_80", "value_80"}, - {"key_81", "value_81"}, - {"key_82", "value_82"}, - {"key_83", "value_83"}, - {"key_84", "value_84"}, - {"key_85", "value_85"}, - {"key_86", "value_86"}, - {"key_87", "value_87"}, - {"key_88", "value_88"}, - {"key_89", "value_89"}, - {"key_90", "value_90"}, - {"key_91", "value_91"}, - {"key_92", "value_92"}, - {"key_93", "value_93"}, - {"key_94", "value_94"}, - {"key_95", "value_95"}, - {"key_96", "value_96"}, - {"key_97", "value_97"}, - {"key_98", "value_98"}, - {"key_99", "value_99"}, + {"key_0", "value_0"}, {"key_1", "value_1"}, {"key_2", "value_2"}, + {"key_3", "value_3"}, {"key_4", "value_4"}, {"key_5", "value_5"}, + {"key_6", "value_6"}, {"key_7", "value_7"}, {"key_8", "value_8"}, + {"key_9", "value_9"}, {"key_10", "value_10"}, {"key_11", "value_11"}, + {"key_12", "value_12"}, {"key_13", "value_13"}, {"key_14", "value_14"}, + {"key_15", "value_15"}, {"key_16", "value_16"}, {"key_17", "value_17"}, + {"key_18", "value_18"}, {"key_19", "value_19"}, {"key_20", "value_20"}, + {"key_21", "value_21"}, {"key_22", "value_22"}, {"key_23", "value_23"}, + {"key_24", "value_24"}, {"key_25", "value_25"}, {"key_26", "value_26"}, + {"key_27", "value_27"}, {"key_28", "value_28"}, {"key_29", "value_29"}, + {"key_30", "value_30"}, {"key_31", "value_31"}, {"key_32", "value_32"}, + {"key_33", "value_33"}, {"key_34", "value_34"}, {"key_35", "value_35"}, + {"key_36", "value_36"}, {"key_37", "value_37"}, {"key_38", "value_38"}, + {"key_39", "value_39"}, {"key_40", "value_40"}, {"key_41", "value_41"}, + {"key_42", "value_42"}, {"key_43", "value_43"}, {"key_44", "value_44"}, + {"key_45", "value_45"}, {"key_46", "value_46"}, {"key_47", "value_47"}, + {"key_48", "value_48"}, {"key_49", "value_49"}, {"key_50", "value_50"}, + {"key_51", "value_51"}, {"key_52", "value_52"}, {"key_53", "value_53"}, + {"key_54", "value_54"}, {"key_55", "value_55"}, {"key_56", "value_56"}, + {"key_57", "value_57"}, {"key_58", "value_58"}, {"key_59", "value_59"}, + {"key_60", "value_60"}, {"key_61", "value_61"}, {"key_62", "value_62"}, + {"key_63", "value_63"}, {"key_64", "value_64"}, {"key_65", "value_65"}, + {"key_66", "value_66"}, {"key_67", "value_67"}, {"key_68", "value_68"}, + {"key_69", "value_69"}, {"key_70", "value_70"}, {"key_71", "value_71"}, + {"key_72", "value_72"}, {"key_73", "value_73"}, {"key_74", "value_74"}, + {"key_75", "value_75"}, {"key_76", "value_76"}, {"key_77", "value_77"}, + {"key_78", "value_78"}, {"key_79", "value_79"}, {"key_80", "value_80"}, + {"key_81", "value_81"}, {"key_82", "value_82"}, {"key_83", "value_83"}, + {"key_84", "value_84"}, {"key_85", "value_85"}, {"key_86", "value_86"}, + {"key_87", "value_87"}, {"key_88", "value_88"}, {"key_89", "value_89"}, + {"key_90", "value_90"}, {"key_91", "value_91"}, {"key_92", "value_92"}, + {"key_93", "value_93"}, {"key_94", "value_94"}, {"key_95", "value_95"}, + {"key_96", "value_96"}, {"key_97", "value_97"}, {"key_98", "value_98"}, + {"key_99", "value_99"}, }; const size_t num_entries = GPR_ARRAY_SIZE(test_entries); // Construct table. @@ -195,7 +129,7 @@ static void test_slice_hash_table() { grpc_exec_ctx_finish(&exec_ctx); } -int main(int argc, char **argv) { +int main(int argc, char** argv) { grpc_test_init(argc, argv); test_slice_hash_table(); return 0; From f186d9d35b93bed6cb47e2902eb3caf4d43be55e Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 20 Apr 2017 07:05:34 -0700 Subject: [PATCH 25/51] Fix asan bug. --- test/core/slice/slice_hash_table_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/slice/slice_hash_table_test.c b/test/core/slice/slice_hash_table_test.c index 9eba57e70aa..055d7f448fd 100644 --- a/test/core/slice/slice_hash_table_test.c +++ b/test/core/slice/slice_hash_table_test.c @@ -50,7 +50,7 @@ typedef struct { static void populate_entries(const test_entry* input, size_t num_entries, grpc_slice_hash_table_entry* output) { for (size_t i = 0; i < num_entries; ++i) { - output[i].key = grpc_slice_from_copied_string(gpr_strdup(input[i].key)); + output[i].key = grpc_slice_from_copied_string(input[i].key); output[i].value = gpr_strdup(input[i].value); } } From 493be1f1b98d3c238b9b87fc478c3f4ae25444af Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 20 Apr 2017 07:10:22 -0700 Subject: [PATCH 26/51] Fix windows build. --- test/core/slice/slice_hash_table_test.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/core/slice/slice_hash_table_test.c b/test/core/slice/slice_hash_table_test.c index 055d7f448fd..67041b2d5c9 100644 --- a/test/core/slice/slice_hash_table_test.c +++ b/test/core/slice/slice_hash_table_test.c @@ -116,10 +116,12 @@ static void test_slice_hash_table() { }; const size_t num_entries = GPR_ARRAY_SIZE(test_entries); // Construct table. - grpc_slice_hash_table_entry entries[num_entries]; + grpc_slice_hash_table_entry* entries = + gpr_zalloc(sizeof(*entries) * num_entries); populate_entries(test_entries, num_entries, entries); grpc_slice_hash_table* table = grpc_slice_hash_table_create(num_entries, entries, destroy_string); + gpr_free(entries); // Check contents of table. check_values(test_entries, num_entries, table); check_non_existent_value("XX", table); From e6f6a09c171cba7398d04c293763b7593ab853e8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 20 Apr 2017 16:50:00 +0000 Subject: [PATCH 27/51] Eliminate gtest splitting: hopefully allows tuning cpu cost better --- build.yaml | 42 ++-------------- tools/run_tests/generated/tests.json | 74 ++++++++++++++-------------- 2 files changed, 42 insertions(+), 74 deletions(-) diff --git a/build.yaml b/build.yaml index f23de87d11c..8c27c8e3da2 100644 --- a/build.yaml +++ b/build.yaml @@ -3114,7 +3114,7 @@ targets: - linux - posix - name: alarm_cpp_test - gtest: true + cpu_cost: 0.1 build: test language: c++ src: @@ -3127,7 +3127,7 @@ targets: - gpr_test_util - gpr - name: async_end2end_test - gtest: true + cpu_cost: 0.8 build: test language: c++ src: @@ -3140,7 +3140,6 @@ targets: - gpr_test_util - gpr - name: auth_property_iterator_test - gtest: true build: test language: c++ src: @@ -3442,6 +3441,7 @@ targets: - linux - posix - name: bm_pollset + cpu_cost: 0.5 build: test language: c++ src: @@ -3463,7 +3463,6 @@ targets: - linux - posix - name: channel_arguments_test - gtest: true build: test language: c++ src: @@ -3473,7 +3472,6 @@ targets: - grpc - gpr - name: channel_filter_test - gtest: true build: test language: c++ src: @@ -3483,7 +3481,6 @@ targets: - grpc - gpr - name: cli_call_test - gtest: true build: test language: c++ src: @@ -3497,7 +3494,6 @@ targets: - gpr_test_util - gpr - name: client_crash_test - gtest: true cpu_cost: 0.1 build: test language: c++ @@ -3528,7 +3524,6 @@ targets: - gpr_test_util - gpr - name: codegen_test_full - gtest: true build: test language: c++ src: @@ -3545,7 +3540,6 @@ targets: filegroups: - grpc++_codegen_base - name: codegen_test_minimal - gtest: true build: test language: c++ src: @@ -3559,7 +3553,6 @@ targets: - grpc++_codegen_base - grpc++_codegen_base_src - name: credentials_test - gtest: true build: test language: c++ src: @@ -3569,7 +3562,6 @@ targets: - grpc - gpr - name: cxx_byte_buffer_test - gtest: true build: test language: c++ src: @@ -3581,7 +3573,6 @@ targets: - gpr_test_util - gpr - name: cxx_slice_test - gtest: true build: test language: c++ src: @@ -3593,7 +3584,6 @@ targets: - gpr_test_util - gpr - name: cxx_string_ref_test - gtest: true build: test language: c++ src: @@ -3601,7 +3591,6 @@ targets: deps: - grpc++ - name: cxx_time_test - gtest: true build: test language: c++ src: @@ -3613,8 +3602,7 @@ targets: - gpr_test_util - gpr - name: end2end_test - gtest: true - cpu_cost: 0.5 + cpu_cost: 1.0 build: test language: c++ src: @@ -3627,7 +3615,6 @@ targets: - gpr_test_util - gpr - name: error_details_test - gtest: true build: test language: c++ src: @@ -3637,7 +3624,6 @@ targets: - grpc++_error_details - grpc++ - name: filter_end2end_test - gtest: true build: test language: c++ src: @@ -3650,7 +3636,6 @@ targets: - gpr_test_util - gpr - name: generic_end2end_test - gtest: true build: test language: c++ src: @@ -3663,7 +3648,6 @@ targets: - gpr_test_util - gpr - name: golden_file_test - gtest: true build: test language: c++ src: @@ -3757,7 +3741,6 @@ targets: vs_config_type: Application vs_project_guid: '{069E9D05-B78B-4751-9252-D21EBAE7DE8E}' - name: grpc_tool_test - gtest: true build: test language: c++ src: @@ -3777,7 +3760,6 @@ targets: filegroups: - grpc++_codegen_proto - name: grpclb_api_test - gtest: true build: test language: c++ src: @@ -3789,9 +3771,9 @@ targets: - grpc++ - grpc - name: grpclb_test - gtest: false build: test language: c++ + cpu_cost: 0.1 src: - src/proto/grpc/lb/v1/load_balancer.proto - test/cpp/grpclb/grpclb_test.cc @@ -3803,7 +3785,6 @@ targets: - gpr_test_util - gpr - name: health_service_end2end_test - gtest: true build: test language: c++ src: @@ -3932,7 +3913,6 @@ targets: - gpr - grpc++_test_config - name: mock_test - gtest: true build: test language: c++ src: @@ -3953,7 +3933,6 @@ targets: - benchmark defaults: benchmark - name: proto_server_reflection_test - gtest: true build: test language: c++ src: @@ -3968,7 +3947,6 @@ targets: - gpr_test_util - gpr - name: proto_utils_test - gtest: true build: test language: c++ src: @@ -4086,7 +4064,6 @@ targets: - gpr - grpc++_test_config - name: round_robin_end2end_test - gtest: true build: test language: c++ src: @@ -4099,7 +4076,6 @@ targets: - gpr_test_util - gpr - name: secure_auth_context_test - gtest: true build: test language: c++ src: @@ -4129,7 +4105,6 @@ targets: - linux - posix - name: server_builder_plugin_test - gtest: true build: test language: c++ src: @@ -4142,7 +4117,6 @@ targets: - gpr_test_util - gpr - name: server_builder_test - gtest: true build: test language: c++ src: @@ -4157,7 +4131,6 @@ targets: - grpc - gpr - name: server_context_test_spouse_test - gtest: true build: test language: c++ src: @@ -4171,7 +4144,6 @@ targets: uses: - grpc++_test - name: server_crash_test - gtest: true cpu_cost: 0.1 build: test language: c++ @@ -4202,7 +4174,6 @@ targets: - gpr_test_util - gpr - name: shutdown_test - gtest: true build: test language: c++ src: @@ -4226,7 +4197,6 @@ targets: - gpr_test_util - gpr - name: streaming_throughput_test - gtest: true build: test language: c++ src: @@ -4280,7 +4250,6 @@ targets: - gpr - grpc++_test_config - name: thread_stress_test - gtest: true cpu_cost: 100 build: test language: c++ @@ -4294,7 +4263,6 @@ targets: - gpr_test_util - gpr - name: writes_per_rpc_test - gtest: true cpu_cost: 0.5 build: test language: c++ diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 6338ea70127..9b48d85970c 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2591,11 +2591,11 @@ "posix", "windows" ], - "cpu_cost": 1.0, + "cpu_cost": 0.1, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "alarm_cpp_test", "platforms": [ @@ -2613,11 +2613,11 @@ "posix", "windows" ], - "cpu_cost": 1.0, + "cpu_cost": 0.6, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "async_end2end_test", "platforms": [ @@ -2639,7 +2639,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "auth_property_iterator_test", "platforms": [ @@ -2989,7 +2989,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "channel_arguments_test", "platforms": [ @@ -3011,7 +3011,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "channel_filter_test", "platforms": [ @@ -3033,7 +3033,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "cli_call_test", "platforms": [ @@ -3054,7 +3054,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "client_crash_test", "platforms": [ @@ -3075,7 +3075,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "codegen_test_full", "platforms": [ @@ -3097,7 +3097,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "codegen_test_minimal", "platforms": [ @@ -3119,7 +3119,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "credentials_test", "platforms": [ @@ -3141,7 +3141,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "cxx_byte_buffer_test", "platforms": [ @@ -3163,7 +3163,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "cxx_slice_test", "platforms": [ @@ -3185,7 +3185,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "cxx_string_ref_test", "platforms": [ @@ -3207,7 +3207,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "cxx_time_test", "platforms": [ @@ -3229,7 +3229,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "end2end_test", "platforms": [ @@ -3251,7 +3251,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "error_details_test", "platforms": [ @@ -3273,7 +3273,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "filter_end2end_test", "platforms": [ @@ -3295,7 +3295,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "generic_end2end_test", "platforms": [ @@ -3319,7 +3319,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "golden_file_test", "platforms": [ @@ -3341,7 +3341,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "grpc_tool_test", "platforms": [ @@ -3363,7 +3363,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "grpclb_api_test", "platforms": [ @@ -3407,7 +3407,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "health_service_end2end_test", "platforms": [ @@ -3471,7 +3471,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "mock_test", "platforms": [ @@ -3515,7 +3515,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "proto_server_reflection_test", "platforms": [ @@ -3537,7 +3537,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "proto_utils_test", "platforms": [ @@ -3579,7 +3579,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "round_robin_end2end_test", "platforms": [ @@ -3601,7 +3601,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "secure_auth_context_test", "platforms": [ @@ -3643,7 +3643,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "server_builder_plugin_test", "platforms": [ @@ -3665,7 +3665,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "server_builder_test", "platforms": [ @@ -3687,7 +3687,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "server_context_test_spouse_test", "platforms": [ @@ -3708,7 +3708,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "server_crash_test", "platforms": [ @@ -3729,7 +3729,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "shutdown_test", "platforms": [ @@ -3772,7 +3772,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "streaming_throughput_test", "platforms": [ @@ -3815,7 +3815,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "thread_stress_test", "platforms": [ @@ -3836,7 +3836,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": true, + "gtest": false, "language": "c++", "name": "writes_per_rpc_test", "platforms": [ From 41fdf7d6ade3987e44213e591a1c67fee81e12a2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 18:55:46 +0000 Subject: [PATCH 28/51] Better cost estimation --- test/cpp/qps/gen_build_yaml.py | 1 + tools/run_tests/generated/tests.json | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py index 2f035abeddc..805b0faeece 100755 --- a/test/cpp/qps/gen_build_yaml.py +++ b/test/cpp/qps/gen_build_yaml.py @@ -66,6 +66,7 @@ def _scenario_json_string(scenario_json, is_tsan): def threads_required(scenario_json, where, is_tsan): scenario_json = mutate_scenario(scenario_json, is_tsan) if scenario_json['%s_config' % where]['%s_type' % where] == 'ASYNC_%s' % where.upper(): + if scenario_json['client_config']['client_channels'] == 1: return 1 return scenario_json['%s_config' % where].get('async_%s_threads' % where, 0) return scenario_json['client_config']['outstanding_rpcs_per_channel'] * scenario_json['client_config']['client_channels'] diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 9b48d85970c..a5bc8fa0ea5 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -41374,7 +41374,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 100, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41476,7 +41476,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41528,7 +41528,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41953,7 +41953,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 100, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42055,7 +42055,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42107,7 +42107,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42580,7 +42580,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 100, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42730,7 +42730,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42806,7 +42806,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43435,7 +43435,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 100, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43585,7 +43585,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43661,7 +43661,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": "capacity", + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", From dbc12105a3e9266dde57647ccc6340ea9c50fd2c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 20 Apr 2017 17:08:39 +0000 Subject: [PATCH 29/51] Refine ping-pong cpu requirement estimations; sort tests by cpu cost to get better bin packing --- build.yaml | 6 ++- test/cpp/qps/gen_build_yaml.py | 2 + tools/run_tests/generated/tests.json | 66 ++++++++++++++-------------- tools/run_tests/run_tests.py | 2 +- 4 files changed, 40 insertions(+), 36 deletions(-) diff --git a/build.yaml b/build.yaml index 8c27c8e3da2..a2bbcaf41db 100644 --- a/build.yaml +++ b/build.yaml @@ -3638,6 +3638,7 @@ targets: - name: generic_end2end_test build: test language: c++ + cpu_cost: 0.1 src: - test/cpp/end2end/generic_end2end_test.cc deps: @@ -3743,6 +3744,7 @@ targets: - name: grpc_tool_test build: test language: c++ + cpu_cost: 0.2 src: - src/proto/grpc/testing/echo.proto - src/proto/grpc/testing/echo_messages.proto @@ -3771,9 +3773,9 @@ targets: - grpc++ - grpc - name: grpclb_test + cpu_cost: 0.1 build: test language: c++ - cpu_cost: 0.1 src: - src/proto/grpc/lb/v1/load_balancer.proto - test/cpp/grpclb/grpclb_test.cc @@ -4263,7 +4265,7 @@ targets: - gpr_test_util - gpr - name: writes_per_rpc_test - cpu_cost: 0.5 + cpu_cost: 0.8 build: test language: c++ src: diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py index 805b0faeece..2edcb86a684 100755 --- a/test/cpp/qps/gen_build_yaml.py +++ b/test/cpp/qps/gen_build_yaml.py @@ -65,6 +65,8 @@ def _scenario_json_string(scenario_json, is_tsan): def threads_required(scenario_json, where, is_tsan): scenario_json = mutate_scenario(scenario_json, is_tsan) + if scenario_json['client_config']['outstanding_rpcs_per_channel'] == 1 and scenario_json['client_config']['client_channels'] == 1: + return 0.4 if scenario_json['%s_config' % where]['%s_type' % where] == 'ASYNC_%s' % where.upper(): if scenario_json['client_config']['client_channels'] == 1: return 1 return scenario_json['%s_config' % where].get('async_%s_threads' % where, 0) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index a5bc8fa0ea5..daa9f5b0e66 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2613,7 +2613,7 @@ "posix", "windows" ], - "cpu_cost": 0.6, + "cpu_cost": 0.8, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -2964,7 +2964,7 @@ "mac", "posix" ], - "cpu_cost": 1.0, + "cpu_cost": 0.5, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -3225,7 +3225,7 @@ "posix", "windows" ], - "cpu_cost": 0.5, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -3381,7 +3381,7 @@ "posix", "windows" ], - "cpu_cost": 1.0, + "cpu_cost": 0.1, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -3832,7 +3832,7 @@ "mac", "posix" ], - "cpu_cost": 0.5, + "cpu_cost": 0.8, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -41274,7 +41274,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41476,7 +41476,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41528,7 +41528,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41553,7 +41553,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41603,7 +41603,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41653,7 +41653,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41753,7 +41753,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41853,7 +41853,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42055,7 +42055,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42107,7 +42107,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42132,7 +42132,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42182,7 +42182,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42232,7 +42232,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42332,7 +42332,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42432,7 +42432,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42730,7 +42730,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42806,7 +42806,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42843,7 +42843,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42917,7 +42917,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42991,7 +42991,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43139,7 +43139,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43287,7 +43287,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43585,7 +43585,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43661,7 +43661,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43698,7 +43698,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43772,7 +43772,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43846,7 +43846,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43994,7 +43994,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": 0.8, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index cfa7071e002..32da7fb02a2 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1466,7 +1466,7 @@ def _build_and_run( # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if args.travis: - massaged_one_run = sorted(one_run, key=lambda x: x.shortname) + massaged_one_run = sorted(one_run, key=lambda x: (x.cpu_cost, x.shortname)) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. From 03876f66e17fc8a614407bb2aa317b21a2ef7319 Mon Sep 17 00:00:00 2001 From: yang-g Date: Thu, 20 Apr 2017 14:51:58 -0700 Subject: [PATCH 30/51] Fix http2_interop test for c++ and python --- src/python/grpcio_tests/tests/http2/negative_http2_client.py | 2 -- test/cpp/interop/http2_client.cc | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/python/grpcio_tests/tests/http2/negative_http2_client.py b/src/python/grpcio_tests/tests/http2/negative_http2_client.py index b184e62cfd4..90f54e80bbf 100644 --- a/src/python/grpcio_tests/tests/http2/negative_http2_client.py +++ b/src/python/grpcio_tests/tests/http2/negative_http2_client.py @@ -96,8 +96,6 @@ def _rst_during_data(stub): def _rst_after_data(stub): resp_future = stub.UnaryCall.future(_SIMPLE_REQUEST) - _validate_payload_type_and_length( - next(resp_future), messages_pb2.COMPRESSABLE, _RESPONSE_SIZE) _validate_status_code_and_details(resp_future, grpc.StatusCode.INTERNAL, "Received RST_STREAM with error code 0") diff --git a/test/cpp/interop/http2_client.cc b/test/cpp/interop/http2_client.cc index 38a437f39f8..2109e346164 100644 --- a/test/cpp/interop/http2_client.cc +++ b/test/cpp/interop/http2_client.cc @@ -108,7 +108,7 @@ bool Http2Client::DoRstAfterData() { SimpleResponse response; AssertStatusCode(SendUnaryCall(&response), grpc::StatusCode::INTERNAL); - GPR_ASSERT(response.has_payload()); // data should be received + // There is no guarantee that data would be received. gpr_log(GPR_DEBUG, "Done testing reset stream after data"); return true; From a935ae1505c768ca0ce6ea981af1fb03c56052d7 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 13 Apr 2017 21:07:45 +0000 Subject: [PATCH 31/51] Fix build rules for grpc++_unsecure, test that it builds properly --- BUILD | 5 ++++- WORKSPACE | 11 +++++------ test/cpp/util/BUILD | 9 +++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/BUILD b/BUILD index 70bd4062a49..57709e47450 100644 --- a/BUILD +++ b/BUILD @@ -1277,6 +1277,9 @@ grpc_cc_library( "src/cpp/server/thread_pool_interface.h", "src/cpp/thread_manager/thread_manager.h", ], + external_deps = [ + "nanopb", + ], language = "c++", public_hdrs = [ "include/grpc++/alarm.h", @@ -1328,7 +1331,7 @@ grpc_cc_library( "include/grpc++/support/time.h", ], deps = [ - "grpc", + "grpc_base", "grpc++_codegen_base", ], ) diff --git a/WORKSPACE b/WORKSPACE index 5ba82f3127b..a78a88979e7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -15,17 +15,17 @@ bind( bind( name = "protobuf", - actual = "@submodule_protobuf//:protobuf", + actual = "@com_google_protobuf//:protobuf", ) bind( name = "protobuf_clib", - actual = "@submodule_protobuf//:protoc_lib", + actual = "@com_google_protobuf//:protoc_lib", ) bind( name = "protocol_compiler", - actual = "@submodule_protobuf//:protoc", + actual = "@com_google_protobuf//:protoc", ) bind( @@ -48,9 +48,8 @@ bind( actual = "@com_github_gflags_gflags//:gflags", ) -new_local_repository( +local_repository( name = "submodule_boringssl", - build_file = "third_party/boringssl-with-bazel/BUILD", path = "third_party/boringssl-with-bazel", ) @@ -61,7 +60,7 @@ new_local_repository( ) new_local_repository( - name = "submodule_protobuf", + name = "com_google_protobuf", build_file = "third_party/protobuf/BUILD", path = "third_party/protobuf", ) diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD index 38f804ffd40..c83f89eb906 100644 --- a/test/cpp/util/BUILD +++ b/test/cpp/util/BUILD @@ -29,6 +29,15 @@ licenses(["notice"]) # 3-clause BSD +# The following builds a shared-object to confirm that grpc++_unsecure +# builds properly. Build-only is sufficient here +cc_binary( + name = "testso.so", + srcs = [], + deps = ["//:grpc++_unsecure"], + linkshared = 1, +) + cc_library( name = "test_config", srcs = [ From f74eaa69575f2570d18dbc37acafb60670f06fe5 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 14 Apr 2017 17:19:52 -0700 Subject: [PATCH 32/51] Fix up protobuf submodule name to match bazel expectation --- bazel/BUILD | 4 ++-- bazel/cc_grpc_library.bzl | 2 +- bazel/generate_cc.bzl | 6 +++--- tools/grpcz/BUILD | 13 +++++++------ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bazel/BUILD b/bazel/BUILD index b86dcc2ad7e..cb2d9d66aee 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -35,12 +35,12 @@ load(":cc_grpc_library.bzl", "cc_grpc_library") proto_library( name = "well_known_protos_list", - srcs = ["@submodule_protobuf//:well_known_protos"], + srcs = ["@com_google_protobuf//:well_known_protos"], ) cc_grpc_library( name = "well_known_protos", srcs = "well_known_protos_list", - deps = [], proto_only = True, + deps = [], ) diff --git a/bazel/cc_grpc_library.bzl b/bazel/cc_grpc_library.bzl index ab1add476e1..a3996eca008 100644 --- a/bazel/cc_grpc_library.bzl +++ b/bazel/cc_grpc_library.bzl @@ -14,7 +14,7 @@ def cc_grpc_library(name, srcs, deps, proto_only, well_known_protos, use_externa the compiled code of any message that the services depend on. well_known_protos: The target from protobuf library that exports well known protos. Currently it will only work if the value is - "@submodule_protobuf//:well_known_protos" + "@com_google_protobuf//:well_known_protos" use_external: When True the grpc deps are prefixed with //external. This allows grpc to be used as a dependency in other bazel projects. **kwargs: rest of arguments, e.g., compatible_with and visibility. diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl index f3961f0a419..35c2983b542 100644 --- a/bazel/generate_cc.bzl +++ b/bazel/generate_cc.bzl @@ -35,10 +35,10 @@ def generate_cc_impl(ctx): well_known_proto_files = [] if ctx.attr.well_known_protos: f = ctx.attr.well_known_protos.files.to_list()[0].dirname - if f != "external/submodule_protobuf/src/google/protobuf": - print("Error: Only @submodule_protobuf//:well_known_protos is supported") + if f != "external/com_google_protobuf/src/google/protobuf": + print("Error: Only @com_google_protobuf//:well_known_protos is supported") else: - # f points to "external/submodule_protobuf/src/google/protobuf" + # f points to "external/com_google_protobuf/src/google/protobuf" # add -I argument to protoc so it knows where to look for the proto files. arguments += ["-I{0}".format(f + "/../..")] well_known_proto_files = [f for f in ctx.attr.well_known_protos.files] diff --git a/tools/grpcz/BUILD b/tools/grpcz/BUILD index cac7df2a9da..cc887a53751 100644 --- a/tools/grpcz/BUILD +++ b/tools/grpcz/BUILD @@ -33,30 +33,31 @@ package(default_visibility = ["//visibility:public"]) load("//:bazel/grpc_build_system.bzl", "grpc_proto_library") -grpc_proto_library ( +grpc_proto_library( name = "monitoring_proto", srcs = [ "monitoring.proto", ], + well_known_protos = "@com_google_protobuf//:well_known_protos", deps = [ ":census_proto", ], - well_known_protos = "@submodule_protobuf//:well_known_protos", ) -grpc_proto_library ( +grpc_proto_library( name = "census_proto", srcs = [ "census.proto", ], - well_known_protos = "@submodule_protobuf//:well_known_protos", + well_known_protos = "@com_google_protobuf//:well_known_protos", ) cc_binary( name = "grpcz_client", - srcs = ["grpcz_client.cc",], + srcs = ["grpcz_client.cc"], deps = [ - "//external:gflags", "monitoring_proto", + "//external:gflags", + "@mongoose_repo//:mongoose_lib", ], ) From 922d117318f1449f17b1f1dc45ca921313efd5eb Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Sun, 16 Apr 2017 15:43:09 -0700 Subject: [PATCH 33/51] One more use of incorrect submodule name --- src/proto/grpc/status/BUILD | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/proto/grpc/status/BUILD b/src/proto/grpc/status/BUILD index c17f87eb3de..71363bd1b63 100644 --- a/src/proto/grpc/status/BUILD +++ b/src/proto/grpc/status/BUILD @@ -37,7 +37,5 @@ grpc_proto_library( name = "status_proto", srcs = ["status.proto"], has_services = False, - well_known_protos = "@submodule_protobuf//:well_known_protos", + well_known_protos = "@com_google_protobuf//:well_known_protos", ) - - From 576e546019140a05cc88f896e7ff50d70704d719 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 20 Apr 2017 18:40:16 +0000 Subject: [PATCH 34/51] Changes required to make build possible: create separate grpc++_base{,_unsecure}, move grpc_compression_trace definition, and remove an assertion about library initialization from server --- BUILD | 23 ++++++++++--------- bazel/grpc_build_system.bzl | 13 +++++++++++ .../message_compress_filter.c | 2 -- src/core/lib/surface/call.c | 1 + src/core/lib/surface/server.c | 2 -- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/BUILD b/BUILD index 57709e47450..09b17ad6eb2 100644 --- a/BUILD +++ b/BUILD @@ -35,7 +35,8 @@ exports_files(["LICENSE"]) package(default_visibility = ["//visibility:public"]) -load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_proto_plugin") +load("//bazel:grpc_build_system.bzl", "grpc_cc_library", + "grpc_proto_plugin", "grpc_cc_libraries") # This should be updated along with build.yaml g_stands_for = "gentle" @@ -163,7 +164,7 @@ grpc_cc_library( standalone = True, deps = [ "gpr", - "grpc++_base", + "grpc++_base_unsecure", "grpc++_codegen_base", "grpc++_codegen_base_src", "grpc_unsecure", @@ -1231,8 +1232,12 @@ grpc_cc_library( ], ) -grpc_cc_library( - name = "grpc++_base", +grpc_cc_libraries( + name_list = ["grpc++_base", "grpc++_base_unsecure"], + additional_dep_list = [ + ["grpc", ], + ["grpc_unsecure", ], + ], srcs = [ "src/cpp/client/channel_cc.cc", "src/cpp/client/client_context.cc", @@ -1267,7 +1272,7 @@ grpc_cc_library( "src/cpp/util/status.cc", "src/cpp/util/string_ref.cc", "src/cpp/util/time_cc.cc", - ], + ], hdrs = [ "src/cpp/client/create_channel_internal.h", "src/cpp/common/channel_filter.h", @@ -1276,10 +1281,7 @@ grpc_cc_library( "src/cpp/server/health/health.pb.h", "src/cpp/server/thread_pool_interface.h", "src/cpp/thread_manager/thread_manager.h", - ], - external_deps = [ - "nanopb", - ], + ], language = "c++", public_hdrs = [ "include/grpc++/alarm.h", @@ -1329,9 +1331,8 @@ grpc_cc_library( "include/grpc++/support/stub_options.h", "include/grpc++/support/sync_stream.h", "include/grpc++/support/time.h", - ], + ], deps = [ - "grpc_base", "grpc++_codegen_base", ], ) diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index 8b524bd0e52..984c06de489 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -49,6 +49,19 @@ def grpc_cc_library(name, srcs = [], public_hdrs = [], hdrs = [], external_deps ] ) +def grpc_cc_libraries(name_list, additional_dep_list, srcs = [], public_hdrs = [], hdrs = [], external_deps = [], deps = [], standalone = False, language="C++"): + for i in range(len(name_list)): + grpc_cc_library( + name = name_list[i], + srcs = srcs, + hdrs = hdrs, + public_hdrs = public_hdrs, + deps = deps + additional_dep_list[i], + external_deps = external_deps, + standalone = standalone, + language = language + ) + def grpc_proto_plugin(name, srcs = [], deps = []): native.cc_binary( name = name, diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.c b/src/core/ext/filters/http/message_compress/message_compress_filter.c index f414a60eee4..4f5f41e9b04 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.c +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.c @@ -49,8 +49,6 @@ #include "src/core/lib/support/string.h" #include "src/core/lib/transport/static_metadata.h" -int grpc_compression_trace = 0; - #define INITIAL_METADATA_UNSEEN 0 #define HAS_COMPRESSION_ALGORITHM 2 #define NO_COMPRESSION_ALGORITHM 4 diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 9aa457d792c..75258065830 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -245,6 +245,7 @@ struct grpc_call { }; int grpc_call_error_trace = 0; +int grpc_compression_trace = 0; #define CALL_STACK_FROM_CALL(call) ((grpc_call_stack *)((call) + 1)) #define CALL_FROM_CALL_STACK(call_stack) (((grpc_call *)(call_stack)) - 1) diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c index 26c81e9aca9..e133d1d2a46 100644 --- a/src/core/lib/surface/server.c +++ b/src/core/lib/surface/server.c @@ -1033,8 +1033,6 @@ grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) { grpc_server *server = gpr_zalloc(sizeof(grpc_server)); - GPR_ASSERT(grpc_is_initialized() && "call grpc_init()"); - gpr_mu_init(&server->mu_global); gpr_mu_init(&server->mu_call); gpr_cv_init(&server->starting_cv); From 2757982c63cfbd5f4e847a06d4a94cbcef8c4e5c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 20 Apr 2017 15:33:40 -0700 Subject: [PATCH 35/51] C++ compatibility fix for tmpfile_posix.c --- src/core/lib/support/tmpfile_posix.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/lib/support/tmpfile_posix.c b/src/core/lib/support/tmpfile_posix.c index 0cd4bb6fc3c..5771c158e07 100644 --- a/src/core/lib/support/tmpfile_posix.c +++ b/src/core/lib/support/tmpfile_posix.c @@ -50,34 +50,34 @@ FILE *gpr_tmpfile(const char *prefix, char **tmp_filename) { FILE *result = NULL; - char *template; + char *filename_template; int fd; if (tmp_filename != NULL) *tmp_filename = NULL; - gpr_asprintf(&template, "/tmp/%s_XXXXXX", prefix); - GPR_ASSERT(template != NULL); + gpr_asprintf(&filename_template, "/tmp/%s_XXXXXX", prefix); + GPR_ASSERT(filename_template != NULL); - fd = mkstemp(template); + fd = mkstemp(filename_template); if (fd == -1) { - gpr_log(GPR_ERROR, "mkstemp failed for template %s with error %s.", - template, strerror(errno)); + gpr_log(GPR_ERROR, "mkstemp failed for filename_template %s with error %s.", + filename_template, strerror(errno)); goto end; } result = fdopen(fd, "w+"); if (result == NULL) { gpr_log(GPR_ERROR, "Could not open file %s from fd %d (error = %s).", - template, fd, strerror(errno)); - unlink(template); + filename_template, fd, strerror(errno)); + unlink(filename_template); close(fd); goto end; } end: if (result != NULL && tmp_filename != NULL) { - *tmp_filename = template; + *tmp_filename = filename_template; } else { - gpr_free(template); + gpr_free(filename_template); } return result; } From cdd9e568641985e25adf4b0b1977e2a53295f77e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 20 Apr 2017 15:47:23 -0700 Subject: [PATCH 36/51] C++ compatibility fix for stack_lockfree.c --- src/core/lib/support/stack_lockfree.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/support/stack_lockfree.c b/src/core/lib/support/stack_lockfree.c index c481a3e0dc5..dfbd3fb1251 100644 --- a/src/core/lib/support/stack_lockfree.c +++ b/src/core/lib/support/stack_lockfree.c @@ -76,13 +76,13 @@ struct gpr_stack_lockfree { gpr_stack_lockfree *gpr_stack_lockfree_create(size_t entries) { gpr_stack_lockfree *stack; - stack = gpr_malloc(sizeof(*stack)); + stack = (gpr_stack_lockfree *)gpr_malloc(sizeof(*stack)); /* Since we only allocate 16 bits to represent an entry number, * make sure that we are within the desired range */ /* Reserve the highest entry number as a dummy */ GPR_ASSERT(entries < INVALID_ENTRY_INDEX); - stack->entries = gpr_malloc_aligned(entries * sizeof(stack->entries[0]), - ENTRY_ALIGNMENT_BITS); + stack->entries = (lockfree_node *)gpr_malloc_aligned( + entries * sizeof(stack->entries[0]), ENTRY_ALIGNMENT_BITS); /* Clear out all entries */ memset(stack->entries, 0, entries * sizeof(stack->entries[0])); memset(&stack->head, 0, sizeof(stack->head)); From fd00263ae91d93e7dea28c6b7f229201e46463b5 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 20 Apr 2017 16:16:49 -0700 Subject: [PATCH 37/51] Fix nit --- src/objective-c/tests/GRPCClientTests.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 0a631d182ab..e36f5c3ee93 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -366,8 +366,8 @@ static GRPCProtoMethod *kUnaryCallMethod; GRXWriter *requestsWriter1 = [GRXWriter writerWithValue:[request data]]; GRPCCall *call1 = [[GRPCCall alloc] initWithHost:kHostAddress - path:kUnaryCallMethod.HTTPPath - requestsWriter:requestsWriter1]; + path:kUnaryCallMethod.HTTPPath + requestsWriter:requestsWriter1]; id responsesWriteable1 = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { NSString *label = [NSString stringWithUTF8String:dispatch_queue_get_label(DISPATCH_CURRENT_QUEUE_LABEL)]; From 26f6c1b2390deab5f891003b448793a63bd09130 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 06:40:29 -0700 Subject: [PATCH 38/51] regen projects --- build.yaml | 4 ++-- tools/run_tests/generated/tests.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.yaml b/build.yaml index a8ba1c8e84c..2ffba8cfa9e 100644 --- a/build.yaml +++ b/build.yaml @@ -3643,9 +3643,9 @@ targets: - gpr_test_util - gpr - name: generic_end2end_test + cpu_cost: 0.1 build: test language: c++ - cpu_cost: 0.1 src: - test/cpp/end2end/generic_end2end_test.cc deps: @@ -3749,9 +3749,9 @@ targets: vs_config_type: Application vs_project_guid: '{069E9D05-B78B-4751-9252-D21EBAE7DE8E}' - name: grpc_tool_test + cpu_cost: 0.2 build: test language: c++ - cpu_cost: 0.2 src: - src/proto/grpc/testing/echo.proto - src/proto/grpc/testing/echo_messages.proto diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index ce4684b286d..38bfb2fcdbb 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -3291,7 +3291,7 @@ "posix", "windows" ], - "cpu_cost": 1.0, + "cpu_cost": 0.1, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -3337,7 +3337,7 @@ "posix", "windows" ], - "cpu_cost": 1.0, + "cpu_cost": 0.2, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, From ab424e8702242c36ee1702d87a11520304e4bb80 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 08:05:40 -0700 Subject: [PATCH 39/51] Back out some bad changes --- Makefile | 9 -- build.yaml | 50 +++++++--- tools/run_tests/generated/configs.json | 3 - tools/run_tests/generated/tests.json | 132 ++++++++----------------- tools/run_tests/run_tests.py | 2 +- 5 files changed, 82 insertions(+), 114 deletions(-) diff --git a/Makefile b/Makefile index 36f2f8ee1d0..31953041d19 100644 --- a/Makefile +++ b/Makefile @@ -157,15 +157,6 @@ LDXX_asan-noleaks = clang++ CPPFLAGS_asan-noleaks = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-noleaks = -fsanitize=address -VALID_CONFIG_c++-compat = 1 -CC_c++-compat = $(DEFAULT_CC) -CXX_c++-compat = $(DEFAULT_CXX) -LD_c++-compat = $(DEFAULT_CC) -LDXX_c++-compat = $(DEFAULT_CXX) -CFLAGS_c++-compat = -Wc++-compat -CPPFLAGS_c++-compat = -O0 -DEFINES_c++-compat = _DEBUG DEBUG - VALID_CONFIG_ubsan = 1 REQUIRE_CUSTOM_LIBRARIES_ubsan = 1 CC_ubsan = clang diff --git a/build.yaml b/build.yaml index 2ffba8cfa9e..045eb6be3b4 100644 --- a/build.yaml +++ b/build.yaml @@ -3118,7 +3118,7 @@ targets: - linux - posix - name: alarm_cpp_test - cpu_cost: 0.1 + gtest: true build: test language: c++ src: @@ -3131,7 +3131,7 @@ targets: - gpr_test_util - gpr - name: async_end2end_test - cpu_cost: 0.8 + gtest: true build: test language: c++ src: @@ -3144,6 +3144,7 @@ targets: - gpr_test_util - gpr - name: auth_property_iterator_test + gtest: true build: test language: c++ src: @@ -3445,7 +3446,6 @@ targets: - linux - posix - name: bm_pollset - cpu_cost: 0.5 build: test language: c++ src: @@ -3467,6 +3467,7 @@ targets: - linux - posix - name: channel_arguments_test + gtest: true build: test language: c++ src: @@ -3476,6 +3477,7 @@ targets: - grpc - gpr - name: channel_filter_test + gtest: true build: test language: c++ src: @@ -3485,6 +3487,7 @@ targets: - grpc - gpr - name: cli_call_test + gtest: true build: test language: c++ src: @@ -3498,6 +3501,7 @@ targets: - gpr_test_util - gpr - name: client_crash_test + gtest: true cpu_cost: 0.1 build: test language: c++ @@ -3528,6 +3532,7 @@ targets: - gpr_test_util - gpr - name: codegen_test_full + gtest: true build: test language: c++ src: @@ -3544,6 +3549,7 @@ targets: filegroups: - grpc++_codegen_base - name: codegen_test_minimal + gtest: true build: test language: c++ src: @@ -3560,6 +3566,7 @@ targets: - grpc++_codegen_base - grpc++_codegen_base_src - name: credentials_test + gtest: true build: test language: c++ src: @@ -3569,6 +3576,7 @@ targets: - grpc - gpr - name: cxx_byte_buffer_test + gtest: true build: test language: c++ src: @@ -3580,6 +3588,7 @@ targets: - gpr_test_util - gpr - name: cxx_slice_test + gtest: true build: test language: c++ src: @@ -3591,6 +3600,7 @@ targets: - gpr_test_util - gpr - name: cxx_string_ref_test + gtest: true build: test language: c++ src: @@ -3598,6 +3608,7 @@ targets: deps: - grpc++ - name: cxx_time_test + gtest: true build: test language: c++ src: @@ -3609,7 +3620,8 @@ targets: - gpr_test_util - gpr - name: end2end_test - cpu_cost: 1.0 + gtest: true + cpu_cost: 0.5 build: test language: c++ src: @@ -3622,6 +3634,7 @@ targets: - gpr_test_util - gpr - name: error_details_test + gtest: true build: test language: c++ src: @@ -3631,6 +3644,7 @@ targets: - grpc++_error_details - grpc++ - name: filter_end2end_test + gtest: true build: test language: c++ src: @@ -3643,7 +3657,7 @@ targets: - gpr_test_util - gpr - name: generic_end2end_test - cpu_cost: 0.1 + gtest: true build: test language: c++ src: @@ -3656,6 +3670,7 @@ targets: - gpr_test_util - gpr - name: golden_file_test + gtest: true build: test language: c++ src: @@ -3749,7 +3764,7 @@ targets: vs_config_type: Application vs_project_guid: '{069E9D05-B78B-4751-9252-D21EBAE7DE8E}' - name: grpc_tool_test - cpu_cost: 0.2 + gtest: true build: test language: c++ src: @@ -3769,6 +3784,7 @@ targets: filegroups: - grpc++_codegen_proto - name: grpclb_api_test + gtest: true build: test language: c++ src: @@ -3780,7 +3796,7 @@ targets: - grpc++ - grpc - name: grpclb_test - cpu_cost: 0.1 + gtest: false build: test language: c++ src: @@ -3794,6 +3810,7 @@ targets: - gpr_test_util - gpr - name: health_service_end2end_test + gtest: true build: test language: c++ src: @@ -3922,6 +3939,7 @@ targets: - gpr - grpc++_test_config - name: mock_test + gtest: true build: test language: c++ src: @@ -3942,6 +3960,7 @@ targets: - benchmark defaults: benchmark - name: proto_server_reflection_test + gtest: true build: test language: c++ src: @@ -3956,6 +3975,7 @@ targets: - gpr_test_util - gpr - name: proto_utils_test + gtest: true build: test language: c++ src: @@ -4073,6 +4093,7 @@ targets: - gpr - grpc++_test_config - name: round_robin_end2end_test + gtest: true build: test language: c++ src: @@ -4085,6 +4106,7 @@ targets: - gpr_test_util - gpr - name: secure_auth_context_test + gtest: true build: test language: c++ src: @@ -4114,6 +4136,7 @@ targets: - linux - posix - name: server_builder_plugin_test + gtest: true build: test language: c++ src: @@ -4126,6 +4149,7 @@ targets: - gpr_test_util - gpr - name: server_builder_test + gtest: true build: test language: c++ src: @@ -4140,6 +4164,7 @@ targets: - grpc - gpr - name: server_context_test_spouse_test + gtest: true build: test language: c++ src: @@ -4153,6 +4178,7 @@ targets: uses: - grpc++_test - name: server_crash_test + gtest: true cpu_cost: 0.1 build: test language: c++ @@ -4183,6 +4209,7 @@ targets: - gpr_test_util - gpr - name: shutdown_test + gtest: true build: test language: c++ src: @@ -4206,6 +4233,7 @@ targets: - gpr_test_util - gpr - name: streaming_throughput_test + gtest: true build: test language: c++ src: @@ -4259,6 +4287,7 @@ targets: - gpr - grpc++_test_config - name: thread_stress_test + gtest: true cpu_cost: 100 build: test language: c++ @@ -4272,7 +4301,8 @@ targets: - gpr_test_util - gpr - name: writes_per_rpc_test - cpu_cost: 0.8 + gtest: true + cpu_cost: 0.5 build: test language: c++ src: @@ -4354,10 +4384,6 @@ configs: basicprof: CPPFLAGS: -O2 -DGRPC_BASIC_PROFILER -DGRPC_TIMERS_RDTSC DEFINES: NDEBUG - c++-compat: - CFLAGS: -Wc++-compat - CPPFLAGS: -O0 - DEFINES: _DEBUG DEBUG counters: CPPFLAGS: -O2 -DGPR_LOW_LEVEL_COUNTERS DEFINES: NDEBUG diff --git a/tools/run_tests/generated/configs.json b/tools/run_tests/generated/configs.json index abbe76d60c2..93dd6fb3d43 100644 --- a/tools/run_tests/generated/configs.json +++ b/tools/run_tests/generated/configs.json @@ -38,9 +38,6 @@ "ASAN_OPTIONS": "detect_leaks=0:color=always" } }, - { - "config": "c++-compat" - }, { "config": "ubsan", "environ": { diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 27b7c0a8da7..cd7be31edb8 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2591,11 +2591,11 @@ "posix", "windows" ], - "cpu_cost": 0.1, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "alarm_cpp_test", "platforms": [ @@ -2613,11 +2613,11 @@ "posix", "windows" ], - "cpu_cost": 0.8, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "async_end2end_test", "platforms": [ @@ -2639,7 +2639,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "auth_property_iterator_test", "platforms": [ @@ -2964,7 +2964,7 @@ "mac", "posix" ], - "cpu_cost": 0.5, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -2989,7 +2989,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "channel_arguments_test", "platforms": [ @@ -3011,7 +3011,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "channel_filter_test", "platforms": [ @@ -3033,7 +3033,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "cli_call_test", "platforms": [ @@ -3054,7 +3054,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "client_crash_test", "platforms": [ @@ -3075,7 +3075,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "codegen_test_full", "platforms": [ @@ -3097,7 +3097,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "codegen_test_minimal", "platforms": [ @@ -3119,7 +3119,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "credentials_test", "platforms": [ @@ -3141,7 +3141,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "cxx_byte_buffer_test", "platforms": [ @@ -3163,7 +3163,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "cxx_slice_test", "platforms": [ @@ -3185,7 +3185,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "cxx_string_ref_test", "platforms": [ @@ -3207,7 +3207,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "cxx_time_test", "platforms": [ @@ -3225,11 +3225,11 @@ "posix", "windows" ], - "cpu_cost": 1.0, + "cpu_cost": 0.5, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "end2end_test", "platforms": [ @@ -3251,7 +3251,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "error_details_test", "platforms": [ @@ -3273,7 +3273,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "filter_end2end_test", "platforms": [ @@ -3291,11 +3291,11 @@ "posix", "windows" ], - "cpu_cost": 0.1, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "generic_end2end_test", "platforms": [ @@ -3319,7 +3319,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "golden_file_test", "platforms": [ @@ -3337,11 +3337,11 @@ "posix", "windows" ], - "cpu_cost": 0.2, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "grpc_tool_test", "platforms": [ @@ -3363,7 +3363,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "grpclb_api_test", "platforms": [ @@ -3381,7 +3381,7 @@ "posix", "windows" ], - "cpu_cost": 0.1, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -3407,7 +3407,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "health_service_end2end_test", "platforms": [ @@ -3471,7 +3471,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "mock_test", "platforms": [ @@ -3515,7 +3515,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "proto_server_reflection_test", "platforms": [ @@ -3537,7 +3537,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "proto_utils_test", "platforms": [ @@ -3579,7 +3579,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "round_robin_end2end_test", "platforms": [ @@ -3601,7 +3601,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "secure_auth_context_test", "platforms": [ @@ -3643,7 +3643,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "server_builder_plugin_test", "platforms": [ @@ -3665,7 +3665,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "server_builder_test", "platforms": [ @@ -3687,7 +3687,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "server_context_test_spouse_test", "platforms": [ @@ -3708,7 +3708,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "server_crash_test", "platforms": [ @@ -3729,7 +3729,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "shutdown_test", "platforms": [ @@ -3772,7 +3772,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "streaming_throughput_test", "platforms": [ @@ -3815,7 +3815,7 @@ "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "thread_stress_test", "platforms": [ @@ -3832,11 +3832,11 @@ "mac", "posix" ], - "cpu_cost": 0.8, + "cpu_cost": 0.5, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, - "gtest": false, + "gtest": true, "language": "c++", "name": "writes_per_rpc_test", "platforms": [ @@ -42392,7 +42392,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42430,7 +42429,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42468,7 +42466,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42506,7 +42503,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42544,7 +42540,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42582,7 +42577,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42620,7 +42614,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42658,7 +42651,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42698,7 +42690,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42736,7 +42727,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42776,7 +42766,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42814,7 +42803,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42852,7 +42840,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42890,7 +42877,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42928,7 +42914,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -42966,7 +42951,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43004,7 +42988,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43042,7 +43025,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43080,7 +43062,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43118,7 +43099,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43156,7 +43136,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43194,7 +43173,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43232,7 +43210,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43270,7 +43247,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43308,7 +43284,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43346,7 +43321,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43384,7 +43358,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43422,7 +43395,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43460,7 +43432,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43498,7 +43469,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43536,7 +43506,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43576,7 +43545,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43614,7 +43582,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43654,7 +43621,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43692,7 +43658,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43730,7 +43695,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43768,7 +43732,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43806,7 +43769,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43844,7 +43806,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43882,7 +43843,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43920,7 +43880,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43958,7 +43917,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -43996,7 +43954,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -44034,7 +43991,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -44072,7 +44028,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", @@ -44110,7 +44065,6 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", - "c++-compat", "counters", "dbg", "gcov", diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index d3e9803a34a..8173cc805c7 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1467,7 +1467,7 @@ def _build_and_run( # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if args.travis: - massaged_one_run = sorted(one_run, key=lambda x: (x.cpu_cost, x.shortname)) + massaged_one_run = sorted(one_run, key=lambda x: x.shortname) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. From 243a12e87a77801ecd21078da52071de9c63d479 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 08:07:14 -0700 Subject: [PATCH 40/51] Reinstate c++-compat --- Makefile | 9 +++++ build.yaml | 4 +++ tools/run_tests/generated/configs.json | 3 ++ tools/run_tests/generated/tests.json | 46 ++++++++++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/Makefile b/Makefile index 31953041d19..36f2f8ee1d0 100644 --- a/Makefile +++ b/Makefile @@ -157,6 +157,15 @@ LDXX_asan-noleaks = clang++ CPPFLAGS_asan-noleaks = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-noleaks = -fsanitize=address +VALID_CONFIG_c++-compat = 1 +CC_c++-compat = $(DEFAULT_CC) +CXX_c++-compat = $(DEFAULT_CXX) +LD_c++-compat = $(DEFAULT_CC) +LDXX_c++-compat = $(DEFAULT_CXX) +CFLAGS_c++-compat = -Wc++-compat +CPPFLAGS_c++-compat = -O0 +DEFINES_c++-compat = _DEBUG DEBUG + VALID_CONFIG_ubsan = 1 REQUIRE_CUSTOM_LIBRARIES_ubsan = 1 CC_ubsan = clang diff --git a/build.yaml b/build.yaml index 045eb6be3b4..ed7f4cce649 100644 --- a/build.yaml +++ b/build.yaml @@ -4384,6 +4384,10 @@ configs: basicprof: CPPFLAGS: -O2 -DGRPC_BASIC_PROFILER -DGRPC_TIMERS_RDTSC DEFINES: NDEBUG + c++-compat: + CFLAGS: -Wc++-compat + CPPFLAGS: -O0 + DEFINES: _DEBUG DEBUG counters: CPPFLAGS: -O2 -DGPR_LOW_LEVEL_COUNTERS DEFINES: NDEBUG diff --git a/tools/run_tests/generated/configs.json b/tools/run_tests/generated/configs.json index 93dd6fb3d43..abbe76d60c2 100644 --- a/tools/run_tests/generated/configs.json +++ b/tools/run_tests/generated/configs.json @@ -38,6 +38,9 @@ "ASAN_OPTIONS": "detect_leaks=0:color=always" } }, + { + "config": "c++-compat" + }, { "config": "ubsan", "environ": { diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index cd7be31edb8..53460fb47af 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -42392,6 +42392,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42429,6 +42430,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42466,6 +42468,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42503,6 +42506,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42540,6 +42544,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42577,6 +42582,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42614,6 +42620,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42651,6 +42658,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42690,6 +42698,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42727,6 +42736,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42766,6 +42776,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42803,6 +42814,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42840,6 +42852,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42877,6 +42890,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42914,6 +42928,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42951,6 +42966,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -42988,6 +43004,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43025,6 +43042,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43062,6 +43080,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43099,6 +43118,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43136,6 +43156,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43173,6 +43194,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43210,6 +43232,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43247,6 +43270,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43284,6 +43308,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43321,6 +43346,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43358,6 +43384,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43395,6 +43422,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43432,6 +43460,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43469,6 +43498,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43506,6 +43536,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43545,6 +43576,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43582,6 +43614,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43621,6 +43654,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43658,6 +43692,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43695,6 +43730,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43732,6 +43768,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43769,6 +43806,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43806,6 +43844,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43843,6 +43882,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43880,6 +43920,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43917,6 +43958,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43954,6 +43996,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -43991,6 +44034,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -44028,6 +44072,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", @@ -44065,6 +44110,7 @@ "asan-noleaks", "asan-trace-cmp", "basicprof", + "c++-compat", "counters", "dbg", "gcov", From 566fd3b36ee4077c9e02aae23bf1bfe13ea0028d Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Thu, 20 Apr 2017 15:53:51 -0700 Subject: [PATCH 41/51] Fix markdown in README.md --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e6d8792d9c6..0edea885185 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ See [Performance dashboard](http://performance-dot-grpc-testing.appspot.com/expl # Repository Structure & Status -This repository contains source code for gRPC libraries for multiple languages written on top of shared C core library [src/core] (src/core). +This repository contains source code for gRPC libraries for multiple languages written on top of shared C core library [src/core](src/core). Libraries in different languages may be in different states of development. We are seeking contributions for all of these libraries. @@ -36,10 +36,9 @@ Libraries in different languages may be in different states of development. We a | C# | [src/csharp](src/csharp) | 1.0 | | Objective-C | [src/objective-c](src/objective-c) | 1.0 | - -Java source code is in the [grpc-java](http://github.com/grpc/grpc-java) repository. -Go source code is in the [grpc-go](http://github.com/grpc/grpc-go) repository. - +Java source code is in the [grpc-java](http://github.com/grpc/grpc-java) +repository. Go source code is in the +[grpc-go](http://github.com/grpc/grpc-go) repository. See [MANIFEST.md](MANIFEST.md) for a listing of top-level items in the repository. From a8749903dd445e3e01a37a9c111b8887915f6124 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 21 Apr 2017 16:44:01 +0000 Subject: [PATCH 42/51] Avoid duplication in build rules by expanding the grpc_cc_libraries rule-creation macro --- BUILD | 55 ++++++++++++++----------------------- bazel/grpc_build_system.bzl | 11 +++++--- 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/BUILD b/BUILD index 09b17ad6eb2..a919a6328a5 100644 --- a/BUILD +++ b/BUILD @@ -54,33 +54,46 @@ grpc_cc_library( ], ) -grpc_cc_library( - name = "grpc", +grpc_cc_libraries( + name_list = ["grpc", "grpc_unsecure",], srcs = [ "src/core/lib/surface/init.c", - "src/core/plugin_registry/grpc_plugin_registry.c", + ], + additional_src_list = [ + [ + "src/core/plugin_registry/grpc_plugin_registry.c", + ], + [ + "src/core/lib/surface/init_unsecure.c", + "src/core/plugin_registry/grpc_unsecure_plugin_registry.c", + ], ], language = "c", standalone = True, deps = [ "census", "grpc_base", - "grpc_lb_policy_grpclb_secure", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", "grpc_load_reporting", "grpc_max_age_filter", - "grpc_resolver_dns_ares", "grpc_resolver_dns_native", "grpc_resolver_sockaddr", - "grpc_secure", "grpc_transport_chttp2_client_insecure", - "grpc_transport_chttp2_client_secure", "grpc_transport_chttp2_server_insecure", - "grpc_transport_chttp2_server_secure", "grpc_message_size_filter", "grpc_deadline_filter", ], + additional_dep_list = [ + [ + "grpc_secure", + "grpc_resolver_dns_ares", + "grpc_lb_policy_grpclb_secure", + "grpc_transport_chttp2_client_secure", + "grpc_transport_chttp2_server_secure", + ], + [], + ], ) grpc_cc_library( @@ -98,32 +111,6 @@ grpc_cc_library( ], ) -grpc_cc_library( - name = "grpc_unsecure", - srcs = [ - "src/core/lib/surface/init.c", - "src/core/lib/surface/init_unsecure.c", - "src/core/plugin_registry/grpc_unsecure_plugin_registry.c", - ], - language = "c", - standalone = True, - deps = [ - "census", - "grpc_base", - "grpc_lb_policy_grpclb", - "grpc_lb_policy_pick_first", - "grpc_lb_policy_round_robin", - "grpc_load_reporting", - "grpc_max_age_filter", - "grpc_resolver_dns_native", - "grpc_resolver_sockaddr", - "grpc_transport_chttp2_client_insecure", - "grpc_transport_chttp2_server_insecure", - "grpc_message_size_filter", - "grpc_deadline_filter", - ], -) - grpc_cc_library( name = "grpc++", srcs = [ diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl index 984c06de489..a438186c757 100644 --- a/bazel/grpc_build_system.bzl +++ b/bazel/grpc_build_system.bzl @@ -49,14 +49,17 @@ def grpc_cc_library(name, srcs = [], public_hdrs = [], hdrs = [], external_deps ] ) -def grpc_cc_libraries(name_list, additional_dep_list, srcs = [], public_hdrs = [], hdrs = [], external_deps = [], deps = [], standalone = False, language="C++"): - for i in range(len(name_list)): +def grpc_cc_libraries(name_list, additional_src_list = [], additional_dep_list = [], srcs = [], public_hdrs = [], hdrs = [], external_deps = [], deps = [], standalone = False, language="C++"): + names = len(name_list) + asl = additional_src_list + [[]]*(names - len(additional_src_list)) + adl = additional_dep_list + [[]]*(names - len(additional_dep_list)) + for i in range(names): grpc_cc_library( name = name_list[i], - srcs = srcs, + srcs = srcs + asl[i], hdrs = hdrs, public_hdrs = public_hdrs, - deps = deps + additional_dep_list[i], + deps = deps + adl[i], external_deps = external_deps, standalone = standalone, language = language From 811fcef8ad385b07135141834f4315b0dd568e9a Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 20 Apr 2017 15:36:32 -0700 Subject: [PATCH 43/51] doc/interop: Fix link for CacheableUnaryCall --- doc/interop-test-descriptions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md index 66a034d6307..050a6234c9d 100644 --- a/doc/interop-test-descriptions.md +++ b/doc/interop-test-descriptions.md @@ -986,6 +986,7 @@ for the `SimpleRequest.response_type`. If the server does not support the `response_type`, then it should fail the RPC with `INVALID_ARGUMENT`. ### CacheableUnaryCall +[CacheableUnaryCall]: #cacheableunarycall Server gets the default SimpleRequest proto as the request. The content of the request is ignored. It returns the SimpleResponse proto with the payload set From 401aa08915b427c03a5616e79127cc0f882bdd8a Mon Sep 17 00:00:00 2001 From: Eric Anderson Date: Thu, 20 Apr 2017 15:37:05 -0700 Subject: [PATCH 44/51] doc/interop: Simplify language to fix missed x-user-ip --- doc/interop-test-descriptions.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md index 050a6234c9d..b040621f883 100644 --- a/doc/interop-test-descriptions.md +++ b/doc/interop-test-descriptions.md @@ -81,9 +81,8 @@ Procedure: Client marks the request as cacheable by setting the cacheable flag in the request context. Longer term this should be driven by the method option specified in the proto file itself. - 2. Client calls CacheableUnaryCall with `SimpleRequest` request again - immediately with the same payload as the previous request. Cacheable flag is - also set for this request's context. + 2. Client calls CacheableUnaryCall again immediately with the same request and + configuration as the previous call. Client asserts: * Both calls were successful From c3fc8342b19e774ef6daf8d4736ca1af61408c75 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 21 Apr 2017 15:01:18 -0700 Subject: [PATCH 45/51] Demote handshake logging msg to DEBUG --- src/core/ext/transport/chttp2/server/chttp2_server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/server/chttp2_server.c b/src/core/ext/transport/chttp2/server/chttp2_server.c index b463506a98f..b9c62c376a1 100644 --- a/src/core/ext/transport/chttp2/server/chttp2_server.c +++ b/src/core/ext/transport/chttp2/server/chttp2_server.c @@ -80,7 +80,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg, gpr_mu_lock(&connection_state->server_state->mu); if (error != GRPC_ERROR_NONE || connection_state->server_state->shutdown) { const char *error_str = grpc_error_string(error); - gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str); + gpr_log(GPR_DEBUG, "Handshaking failed: %s", error_str); if (error == GRPC_ERROR_NONE && args->endpoint != NULL) { // We were shut down after handshaking completed successfully, so From 20cb627339c20e86814d64ba4837d7bdd6f35195 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 15:07:13 -0700 Subject: [PATCH 46/51] Fix HTTP proxy tests - heap allocate the pollset shutdown closure (this may be called asynchronously) - ensure a poller remains until all endpoints are closed --- .../end2end/fixtures/http_proxy_fixture.c | 59 +++++++++++++------ test/core/end2end/tests/cancel_after_invoke.c | 9 +-- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/test/core/end2end/fixtures/http_proxy_fixture.c b/test/core/end2end/fixtures/http_proxy_fixture.c index 451ed268d30..02235d96e73 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.c +++ b/test/core/end2end/fixtures/http_proxy_fixture.c @@ -59,6 +59,7 @@ #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/iomgr/tcp_client.h" #include "src/core/lib/iomgr/tcp_server.h" +#include "src/core/lib/iomgr/timer.h" #include "src/core/lib/slice/slice_internal.h" #include "test/core/util/port.h" @@ -69,7 +70,7 @@ struct grpc_end2end_http_proxy { grpc_channel_args* channel_args; gpr_mu* mu; grpc_pollset* pollset; - gpr_atm shutdown; + gpr_refcount users; }; // @@ -77,6 +78,8 @@ struct grpc_end2end_http_proxy { // typedef struct proxy_connection { + grpc_end2end_http_proxy* proxy; + grpc_endpoint* client_endpoint; grpc_endpoint* server_endpoint; @@ -103,13 +106,26 @@ typedef struct proxy_connection { grpc_http_request http_request; } proxy_connection; +static void proxy_connection_ref(proxy_connection* conn, const char* reason) { + gpr_log(GPR_DEBUG, "proxy_connection_ref: %p %s %" PRIdPTR " --> %" PRIdPTR, + conn, reason, gpr_atm_no_barrier_load(&conn->refcount.count), + gpr_atm_no_barrier_load(&conn->refcount.count) - 1); + gpr_ref(&conn->refcount); +} + // Helper function to destroy the proxy connection. static void proxy_connection_unref(grpc_exec_ctx* exec_ctx, - proxy_connection* conn) { + proxy_connection* conn, const char* reason) { + gpr_log(GPR_DEBUG, "proxy_connection_unref: %p %s %" PRIdPTR " --> %" PRIdPTR, + conn, reason, gpr_atm_no_barrier_load(&conn->refcount.count), + gpr_atm_no_barrier_load(&conn->refcount.count) - 1); if (gpr_unref(&conn->refcount)) { + gpr_log(GPR_DEBUG, "endpoints: %p %p", conn->client_endpoint, + conn->server_endpoint); grpc_endpoint_destroy(exec_ctx, conn->client_endpoint); - if (conn->server_endpoint != NULL) + if (conn->server_endpoint != NULL) { grpc_endpoint_destroy(exec_ctx, conn->server_endpoint); + } grpc_pollset_set_destroy(exec_ctx, conn->pollset_set); grpc_slice_buffer_destroy_internal(exec_ctx, &conn->client_read_buffer); grpc_slice_buffer_destroy_internal(exec_ctx, @@ -121,6 +137,7 @@ static void proxy_connection_unref(grpc_exec_ctx* exec_ctx, grpc_slice_buffer_destroy_internal(exec_ctx, &conn->server_write_buffer); grpc_http_parser_destroy(&conn->http_parser); grpc_http_request_destroy(&conn->http_request); + gpr_unref(&conn->proxy->users); gpr_free(conn); } } @@ -139,7 +156,7 @@ static void proxy_connection_failed(grpc_exec_ctx* exec_ctx, grpc_endpoint_shutdown(exec_ctx, conn->server_endpoint, GRPC_ERROR_REF(error)); } - proxy_connection_unref(exec_ctx, conn); + proxy_connection_unref(exec_ctx, conn, "conn_failed"); } // Callback for writing proxy data to the client. @@ -163,7 +180,7 @@ static void on_client_write_done(grpc_exec_ctx* exec_ctx, void* arg, &conn->on_client_write_done); } else { // No more writes. Unref the connection. - proxy_connection_unref(exec_ctx, conn); + proxy_connection_unref(exec_ctx, conn, "write_done"); } } @@ -188,7 +205,7 @@ static void on_server_write_done(grpc_exec_ctx* exec_ctx, void* arg, &conn->on_server_write_done); } else { // No more writes. Unref the connection. - proxy_connection_unref(exec_ctx, conn); + proxy_connection_unref(exec_ctx, conn, "server_write"); } } @@ -214,7 +231,7 @@ static void on_client_read_done(grpc_exec_ctx* exec_ctx, void* arg, } else { grpc_slice_buffer_move_into(&conn->client_read_buffer, &conn->server_write_buffer); - gpr_ref(&conn->refcount); + proxy_connection_ref(conn, "client_read"); grpc_endpoint_write(exec_ctx, conn->server_endpoint, &conn->server_write_buffer, &conn->on_server_write_done); @@ -246,7 +263,7 @@ static void on_server_read_done(grpc_exec_ctx* exec_ctx, void* arg, } else { grpc_slice_buffer_move_into(&conn->server_read_buffer, &conn->client_write_buffer); - gpr_ref(&conn->refcount); + proxy_connection_ref(conn, "server_read"); grpc_endpoint_write(exec_ctx, conn->client_endpoint, &conn->client_write_buffer, &conn->on_client_write_done); @@ -270,7 +287,9 @@ static void on_write_response_done(grpc_exec_ctx* exec_ctx, void* arg, // Start reading from both client and server. One of the read // requests inherits our ref to conn, but we need to take a new ref // for the other one. - gpr_ref(&conn->refcount); + proxy_connection_ref(conn, "client_read"); + proxy_connection_ref(conn, "server_read"); + proxy_connection_unref(exec_ctx, conn, "write_response"); grpc_endpoint_read(exec_ctx, conn->client_endpoint, &conn->client_read_buffer, &conn->on_client_read_done); grpc_endpoint_read(exec_ctx, conn->server_endpoint, &conn->server_read_buffer, @@ -312,6 +331,8 @@ static void on_server_connect_done(grpc_exec_ctx* exec_ctx, void* arg, static void on_read_request_done(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) { proxy_connection* conn = arg; + gpr_log(GPR_DEBUG, "on_read_request_done: %p %s", conn, + grpc_error_string(error)); if (error != GRPC_ERROR_NONE) { proxy_connection_failed(exec_ctx, conn, true /* is_client */, "HTTP proxy read request", error); @@ -376,12 +397,15 @@ static void on_accept(grpc_exec_ctx* exec_ctx, void* arg, gpr_free(acceptor); grpc_end2end_http_proxy* proxy = arg; // Instantiate proxy_connection. - proxy_connection* conn = gpr_malloc(sizeof(*conn)); - memset(conn, 0, sizeof(*conn)); + proxy_connection* conn = gpr_zalloc(sizeof(*conn)); + gpr_ref(&proxy->users); conn->client_endpoint = endpoint; + conn->proxy = proxy; gpr_ref_init(&conn->refcount, 1); conn->pollset_set = grpc_pollset_set_create(); + gpr_log(GPR_DEBUG, "on_accept: %p", conn); grpc_pollset_set_add_pollset(exec_ctx, conn->pollset_set, proxy->pollset); + grpc_endpoint_add_to_pollset_set(exec_ctx, endpoint, conn->pollset_set); grpc_closure_init(&conn->on_read_request_done, on_read_request_done, conn, grpc_schedule_on_exec_ctx); grpc_closure_init(&conn->on_server_connect_done, on_server_connect_done, conn, @@ -416,6 +440,7 @@ static void thread_main(void* arg) { grpc_end2end_http_proxy* proxy = arg; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; do { + gpr_ref(&proxy->users); const gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); const gpr_timespec deadline = gpr_time_add(now, gpr_time_from_seconds(1, GPR_TIMESPAN)); @@ -426,7 +451,7 @@ static void thread_main(void* arg) { grpc_pollset_work(&exec_ctx, proxy->pollset, &worker, now, deadline)); gpr_mu_unlock(proxy->mu); grpc_exec_ctx_flush(&exec_ctx); - } while (!gpr_atm_acq_load(&proxy->shutdown)); + } while (!gpr_unref(&proxy->users)); grpc_exec_ctx_finish(&exec_ctx); } @@ -434,6 +459,7 @@ grpc_end2end_http_proxy* grpc_end2end_http_proxy_create(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_end2end_http_proxy* proxy = gpr_malloc(sizeof(*proxy)); memset(proxy, 0, sizeof(*proxy)); + gpr_ref_init(&proxy->users, 1); // Construct proxy address. const int proxy_port = grpc_pick_unused_port_or_die(); gpr_join_host_port(&proxy->proxy_name, "localhost", proxy_port); @@ -474,17 +500,16 @@ static void destroy_pollset(grpc_exec_ctx* exec_ctx, void* arg, } void grpc_end2end_http_proxy_destroy(grpc_end2end_http_proxy* proxy) { - gpr_atm_rel_store(&proxy->shutdown, 1); // Signal proxy thread to shutdown. + gpr_unref(&proxy->users); // Signal proxy thread to shutdown. grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; gpr_thd_join(proxy->thd); grpc_tcp_server_shutdown_listeners(&exec_ctx, proxy->server); grpc_tcp_server_unref(&exec_ctx, proxy->server); gpr_free(proxy->proxy_name); grpc_channel_args_destroy(&exec_ctx, proxy->channel_args); - grpc_closure destroyed; - grpc_closure_init(&destroyed, destroy_pollset, proxy->pollset, - grpc_schedule_on_exec_ctx); - grpc_pollset_shutdown(&exec_ctx, proxy->pollset, &destroyed); + grpc_pollset_shutdown(&exec_ctx, proxy->pollset, + grpc_closure_create(destroy_pollset, proxy->pollset, + grpc_schedule_on_exec_ctx)); gpr_free(proxy); grpc_exec_ctx_finish(&exec_ctx); } diff --git a/test/core/end2end/tests/cancel_after_invoke.c b/test/core/end2end/tests/cancel_after_invoke.c index f2aca737ab3..5bc9ed283bb 100644 --- a/test/core/end2end/tests/cancel_after_invoke.c +++ b/test/core/end2end/tests/cancel_after_invoke.c @@ -49,11 +49,12 @@ static void *tag(intptr_t t) { return (void *)t; } static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, const char *test_name, cancellation_mode mode, + size_t test_ops, grpc_channel_args *client_args, grpc_channel_args *server_args) { grpc_end2end_test_fixture f; - gpr_log(GPR_INFO, "Running test: %s/%s/%s", test_name, config.name, - mode.name); + gpr_log(GPR_INFO, "Running test: %s/%s/%s [%" PRIdPTR " ops]", test_name, + config.name, mode.name, test_ops); f = config.create_fixture(client_args, server_args); config.init_server(&f, server_args); config.init_client(&f, client_args); @@ -108,8 +109,8 @@ static void test_cancel_after_invoke(grpc_end2end_test_config config, grpc_op ops[6]; grpc_op *op; grpc_call *c; - grpc_end2end_test_fixture f = - begin_test(config, "test_cancel_after_invoke", mode, NULL, NULL); + grpc_end2end_test_fixture f = begin_test(config, "test_cancel_after_invoke", + mode, test_ops, NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_metadata_array initial_metadata_recv; grpc_metadata_array trailing_metadata_recv; From add943869a8d043ea9e07a58725d5b110650efe8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 15:43:55 -0700 Subject: [PATCH 47/51] Diff now too --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 09b62ae1c90..6ee4cbfc7bb 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -56,6 +56,7 @@ _INTERESTING = ( 'writes_per_iteration', 'atm_cas_per_iteration', 'atm_add_per_iteration', + 'nows_per_iteration', ) def changed_ratio(n, o): From b2b4122c03271a16898c872b13986914ba424116 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 19:05:59 -0700 Subject: [PATCH 48/51] Remove logging --- test/core/end2end/fixtures/http_proxy_fixture.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/core/end2end/fixtures/http_proxy_fixture.c b/test/core/end2end/fixtures/http_proxy_fixture.c index 02235d96e73..a9d71b5ba40 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.c +++ b/test/core/end2end/fixtures/http_proxy_fixture.c @@ -107,18 +107,12 @@ typedef struct proxy_connection { } proxy_connection; static void proxy_connection_ref(proxy_connection* conn, const char* reason) { - gpr_log(GPR_DEBUG, "proxy_connection_ref: %p %s %" PRIdPTR " --> %" PRIdPTR, - conn, reason, gpr_atm_no_barrier_load(&conn->refcount.count), - gpr_atm_no_barrier_load(&conn->refcount.count) - 1); gpr_ref(&conn->refcount); } // Helper function to destroy the proxy connection. static void proxy_connection_unref(grpc_exec_ctx* exec_ctx, proxy_connection* conn, const char* reason) { - gpr_log(GPR_DEBUG, "proxy_connection_unref: %p %s %" PRIdPTR " --> %" PRIdPTR, - conn, reason, gpr_atm_no_barrier_load(&conn->refcount.count), - gpr_atm_no_barrier_load(&conn->refcount.count) - 1); if (gpr_unref(&conn->refcount)) { gpr_log(GPR_DEBUG, "endpoints: %p %p", conn->client_endpoint, conn->server_endpoint); From 4a1925444d81ba09878c55d8576cf981d601990c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 19:07:00 -0700 Subject: [PATCH 49/51] Remove more spam --- test/core/end2end/fixtures/http_proxy_fixture.c | 1 - 1 file changed, 1 deletion(-) diff --git a/test/core/end2end/fixtures/http_proxy_fixture.c b/test/core/end2end/fixtures/http_proxy_fixture.c index a9d71b5ba40..f0d09487c62 100644 --- a/test/core/end2end/fixtures/http_proxy_fixture.c +++ b/test/core/end2end/fixtures/http_proxy_fixture.c @@ -397,7 +397,6 @@ static void on_accept(grpc_exec_ctx* exec_ctx, void* arg, conn->proxy = proxy; gpr_ref_init(&conn->refcount, 1); conn->pollset_set = grpc_pollset_set_create(); - gpr_log(GPR_DEBUG, "on_accept: %p", conn); grpc_pollset_set_add_pollset(exec_ctx, conn->pollset_set, proxy->pollset); grpc_endpoint_add_to_pollset_set(exec_ctx, endpoint, conn->pollset_set); grpc_closure_init(&conn->on_read_request_done, on_read_request_done, conn, From 0d4628479995be4e310f55683f1ebaae2a879e4d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 08:24:21 -0700 Subject: [PATCH 50/51] Rollback some changes --- build.yaml | 2 +- test/cpp/qps/gen_build_yaml.py | 2 - tools/run_tests/generated/tests.json | 58 ++++++++++++++-------------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/build.yaml b/build.yaml index 58a474aabf1..f2af370522e 100644 --- a/build.yaml +++ b/build.yaml @@ -3806,7 +3806,7 @@ targets: - grpc++ - grpc - name: grpclb_test - gtest: false + cpu_cost: 0.1 build: test language: c++ src: diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py index 2edcb86a684..805b0faeece 100755 --- a/test/cpp/qps/gen_build_yaml.py +++ b/test/cpp/qps/gen_build_yaml.py @@ -65,8 +65,6 @@ def _scenario_json_string(scenario_json, is_tsan): def threads_required(scenario_json, where, is_tsan): scenario_json = mutate_scenario(scenario_json, is_tsan) - if scenario_json['client_config']['outstanding_rpcs_per_channel'] == 1 and scenario_json['client_config']['client_channels'] == 1: - return 0.4 if scenario_json['%s_config' % where]['%s_type' % where] == 'ASYNC_%s' % where.upper(): if scenario_json['client_config']['client_channels'] == 1: return 1 return scenario_json['%s_config' % where].get('async_%s_threads' % where, 0) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 203f51b6b89..cb918a0830a 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -3403,7 +3403,7 @@ "posix", "windows" ], - "cpu_cost": 1.0, + "cpu_cost": 0.1, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -41250,7 +41250,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41452,7 +41452,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41504,7 +41504,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41529,7 +41529,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41579,7 +41579,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41629,7 +41629,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41729,7 +41729,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41829,7 +41829,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42031,7 +42031,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42083,7 +42083,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42108,7 +42108,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42158,7 +42158,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42208,7 +42208,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42308,7 +42308,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42408,7 +42408,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42714,7 +42714,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42792,7 +42792,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42830,7 +42830,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42906,7 +42906,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42982,7 +42982,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43134,7 +43134,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43286,7 +43286,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43592,7 +43592,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43670,7 +43670,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43708,7 +43708,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43784,7 +43784,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43860,7 +43860,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -44012,7 +44012,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 0.8, + "cpu_cost": 2, "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", From c6f6663fb7b8c4b1614b6f881abf966d48cb87c4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 08:29:39 -0700 Subject: [PATCH 51/51] Rollback some changes --- build.yaml | 2 +- test/cpp/qps/gen_build_yaml.py | 1 - tools/run_tests/generated/tests.json | 26 +++++++++++++------------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/build.yaml b/build.yaml index f2af370522e..58a474aabf1 100644 --- a/build.yaml +++ b/build.yaml @@ -3806,7 +3806,7 @@ targets: - grpc++ - grpc - name: grpclb_test - cpu_cost: 0.1 + gtest: false build: test language: c++ src: diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py index 805b0faeece..2f035abeddc 100755 --- a/test/cpp/qps/gen_build_yaml.py +++ b/test/cpp/qps/gen_build_yaml.py @@ -66,7 +66,6 @@ def _scenario_json_string(scenario_json, is_tsan): def threads_required(scenario_json, where, is_tsan): scenario_json = mutate_scenario(scenario_json, is_tsan) if scenario_json['%s_config' % where]['%s_type' % where] == 'ASYNC_%s' % where.upper(): - if scenario_json['client_config']['client_channels'] == 1: return 1 return scenario_json['%s_config' % where].get('async_%s_threads' % where, 0) return scenario_json['client_config']['outstanding_rpcs_per_channel'] * scenario_json['client_config']['client_channels'] diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index cb918a0830a..120a84e8a4b 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -3403,7 +3403,7 @@ "posix", "windows" ], - "cpu_cost": 0.1, + "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], "flaky": false, @@ -41350,7 +41350,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 100, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41452,7 +41452,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41504,7 +41504,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -41929,7 +41929,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 100, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42031,7 +42031,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42083,7 +42083,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "tsan", @@ -42560,7 +42560,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 100, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42714,7 +42714,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -42792,7 +42792,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43438,7 +43438,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 100, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43592,7 +43592,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks", @@ -43670,7 +43670,7 @@ "ci_platforms": [ "linux" ], - "cpu_cost": 2, + "cpu_cost": "capacity", "defaults": "boringssl", "exclude_configs": [ "asan-noleaks",