From 37268d0526036fcd29a319932f734a53ee35ae34 Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Tue, 7 Mar 2017 09:22:04 -0800 Subject: [PATCH 01/90] Fix for using gem build/install When pre-compiled assets are missing for the current platform, and you run gem install, the compiled native extensions are placed in the first require paths folder. This differs from rake-compiler task, where they are always placed in lib. Setting the lib path as the first require path will ensure the compiled library is placed in the correct folder. --- grpc.gemspec | 2 +- templates/grpc.gemspec.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grpc.gemspec b/grpc.gemspec index 8d5b7b2ab1c..1d2ee0725a8 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -24,7 +24,7 @@ Gem::Specification.new do |s| s.files += Dir.glob('include/grpc/**/*') s.test_files = Dir.glob('src/ruby/spec/**/*') s.bindir = 'src/ruby/bin' - s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb ) + s.require_paths = %w( src/ruby/lib src/ruby/bin src/ruby/pb ) s.platform = Gem::Platform::RUBY s.add_dependency 'google-protobuf', '~> 3.1' diff --git a/templates/grpc.gemspec.template b/templates/grpc.gemspec.template index 462ea526140..80ce643d802 100644 --- a/templates/grpc.gemspec.template +++ b/templates/grpc.gemspec.template @@ -26,7 +26,7 @@ s.files += Dir.glob('include/grpc/**/*') s.test_files = Dir.glob('src/ruby/spec/**/*') s.bindir = 'src/ruby/bin' - s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb ) + s.require_paths = %w( src/ruby/lib src/ruby/bin src/ruby/pb ) s.platform = Gem::Platform::RUBY s.add_dependency 'google-protobuf', '~> 3.1' From 033c8cb4751b6580fcff02fa12be6e2dd7dfa8fd Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 14 Mar 2017 13:00:11 -0700 Subject: [PATCH 02/90] Advance ProtoCompiler version --- src/objective-c/!ProtoCompiler.podspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index dc4d8e964e6..2e9b944f33d 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.1.0' + v = '3.2.0' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC @@ -110,7 +110,7 @@ Pod::Spec.new do |s| # Restrict the protobuf runtime version to the one supported by this version of protoc. s.dependency 'Protobuf', '~> 3.0' # For the Protobuf dependency not to complain: - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' # This is only for local development of protoc: If the Podfile brings this pod from a local From 082b286b77b0f925cb630030a551a900444bcb34 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 16 Mar 2017 17:13:05 -0700 Subject: [PATCH 03/90] Advance dependency version --- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index ab8f82a39e7..f91f42359f9 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.1.0' + s.dependency '!ProtoCompiler', '3.2.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.1' s.osx.deployment_target = '10.9' From 60cbf8fc391a6d1d3ce37b42e95c3439dace9b4b Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 28 Feb 2017 16:14:40 -0800 Subject: [PATCH 04/90] Relieve ios deployment version to 7.0 --- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- templates/gRPC-Core.podspec.template | 2 +- templates/gRPC-ProtoRPC.podspec.template | 2 +- templates/gRPC-RxLibrary.podspec.template | 2 +- templates/gRPC.podspec.template | 2 +- .../src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 027babcda46..a3010d5e37a 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -51,7 +51,7 @@ Pod::Spec.new do |s| :submodules => true, } - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' s.requires_arc = false diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 6ca9fcd3426..1940015782e 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -48,7 +48,7 @@ Pod::Spec.new do |s| :tag => "v#{version}", } - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' name = 'ProtoRPC' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 51b52c0c440..841a8533633 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -48,7 +48,7 @@ Pod::Spec.new do |s| :tag => "v#{version}", } - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' name = 'RxLibrary' diff --git a/gRPC.podspec b/gRPC.podspec index 83a86803ebc..08ea79118d1 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -47,7 +47,7 @@ Pod::Spec.new do |s| :tag => "v#{version}", } - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' name = 'GRPCClient' diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index f91f42359f9..2f41ad196a6 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -103,7 +103,7 @@ Pod::Spec.new do |s| # Restrict the protoc version to the one supported by this plugin. s.dependency '!ProtoCompiler', '3.2.0' # For the Protobuf dependency not to complain: - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' # Restrict the gRPC runtime version to the one supported by this plugin. s.dependency 'gRPC-ProtoRPC', v diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 9ed32e31cff..11146a490b8 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -78,7 +78,7 @@ :submodules => true, } - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' s.requires_arc = false diff --git a/templates/gRPC-ProtoRPC.podspec.template b/templates/gRPC-ProtoRPC.podspec.template index 5d7d90d2318..47b22dd2a52 100644 --- a/templates/gRPC-ProtoRPC.podspec.template +++ b/templates/gRPC-ProtoRPC.podspec.template @@ -50,7 +50,7 @@ :tag => "v#{version}", } - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' name = 'ProtoRPC' diff --git a/templates/gRPC-RxLibrary.podspec.template b/templates/gRPC-RxLibrary.podspec.template index 35a06c8a856..48f0df8f9e6 100644 --- a/templates/gRPC-RxLibrary.podspec.template +++ b/templates/gRPC-RxLibrary.podspec.template @@ -50,7 +50,7 @@ :tag => "v#{version}", } - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' name = 'RxLibrary' diff --git a/templates/gRPC.podspec.template b/templates/gRPC.podspec.template index d33ce277dc3..ce473608dd8 100644 --- a/templates/gRPC.podspec.template +++ b/templates/gRPC.podspec.template @@ -49,7 +49,7 @@ :tag => "v#{version}", } - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' name = 'GRPCClient' diff --git a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template index 3a10cfab3cc..b100fa7cfe1 100644 --- a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template +++ b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template @@ -105,7 +105,7 @@ # Restrict the protoc version to the one supported by this plugin. s.dependency '!ProtoCompiler', '3.1.0' # For the Protobuf dependency not to complain: - s.ios.deployment_target = '7.1' + s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' # Restrict the gRPC runtime version to the one supported by this plugin. s.dependency 'gRPC-ProtoRPC', v From c007888dccf9ebb7b3240db55fc04aeb0dff9dd1 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 16 Mar 2017 20:53:41 -0700 Subject: [PATCH 05/90] Changes to podspecs --- gRPC-Core.podspec | 7 +++---- templates/gRPC-Core.podspec.template | 7 +++---- .../objective-c/!ProtoCompiler-gRPCPlugin.podspec.template | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index a3010d5e37a..73f327460f4 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -887,8 +887,7 @@ Pod::Spec.new do |s| s.subspec 'Cronet-Interface' do |ss| ss.header_mappings_dir = 'include/grpc' - ss.source_files = 'include/grpc/grpc_cronet.h', - 'src/core/ext/transport/cronet/transport/cronet_transport.h' + ss.source_files = 'include/grpc/grpc_cronet.h' end s.subspec 'Cronet-Implementation' do |ss| @@ -899,7 +898,7 @@ Pod::Spec.new do |s| ss.dependency "#{s.name}/Cronet-Interface", version ss.source_files = 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', - 'src/core/ext/transport/cronet/transport/cronet_transport.c', + 'src/core/ext/transport/cronet/transport/cronet_transport.{c,h}', 'third_party/objective_c/Cronet/bidirectional_stream_c.h' end @@ -914,7 +913,7 @@ Pod::Spec.new do |s| 'test/core/end2end/end2end_test_utils.c', 'test/core/end2end/tests/*.{c,h}', 'test/core/end2end/data/*.{c,h}', - 'test/core/util/debugger_macros.c', + 'test/core/util/debugger_macros.{c,h}', 'test/core/util/test_config.{c,h}', 'test/core/util/port.h', 'test/core/util/port.c', diff --git a/templates/gRPC-Core.podspec.template b/templates/gRPC-Core.podspec.template index 11146a490b8..e7289111b24 100644 --- a/templates/gRPC-Core.podspec.template +++ b/templates/gRPC-Core.podspec.template @@ -161,8 +161,7 @@ s.subspec 'Cronet-Interface' do |ss| ss.header_mappings_dir = 'include/grpc' - ss.source_files = 'include/grpc/grpc_cronet.h', - 'src/core/ext/transport/cronet/transport/cronet_transport.h' + ss.source_files = 'include/grpc/grpc_cronet.h' end s.subspec 'Cronet-Implementation' do |ss| @@ -173,7 +172,7 @@ ss.dependency "#{s.name}/Cronet-Interface", version ss.source_files = 'src/core/ext/transport/cronet/client/secure/cronet_channel_create.c', - 'src/core/ext/transport/cronet/transport/cronet_transport.c', + 'src/core/ext/transport/cronet/transport/cronet_transport.{c,h}', 'third_party/objective_c/Cronet/bidirectional_stream_c.h' end @@ -188,7 +187,7 @@ 'test/core/end2end/end2end_test_utils.c', 'test/core/end2end/tests/*.{c,h}', 'test/core/end2end/data/*.{c,h}', - 'test/core/util/debugger_macros.c', + 'test/core/util/debugger_macros.{c,h}', 'test/core/util/test_config.{c,h}', 'test/core/util/port.h', 'test/core/util/port.c', diff --git a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template index b100fa7cfe1..d7d84f3c774 100644 --- a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template +++ b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template @@ -103,7 +103,7 @@ s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.1.0' + s.dependency '!ProtoCompiler', '3.2.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' From b5b5f020a38c40e4753ead360151810479b8c664 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Thu, 16 Mar 2017 16:42:10 -0700 Subject: [PATCH 06/90] Upgrade Node dependency on Protobuf.js to version 6 --- package.json | 2 +- src/node/index.js | 74 +++++----- src/node/interop/interop_server.js | 2 +- src/node/performance/benchmark_server.js | 2 +- src/node/performance/worker.js | 4 +- src/node/src/client.js | 22 +-- src/node/src/common.js | 115 ++-------------- src/node/src/protobuf_js_5_common.js | 167 +++++++++++++++++++++++ src/node/src/protobuf_js_6_common.js | 154 +++++++++++++++++++++ src/node/src/server.js | 24 +++- src/node/stress/metrics_server.js | 2 +- src/node/test/common_test.js | 35 +++-- src/node/test/credentials_test.js | 2 +- src/node/test/surface_test.js | 116 ++++++++++------ templates/package.json.template | 2 +- 15 files changed, 493 insertions(+), 230 deletions(-) create mode 100644 src/node/src/protobuf_js_5_common.js create mode 100644 src/node/src/protobuf_js_6_common.js diff --git a/package.json b/package.json index 8c0854b1d28..4b216a87ab8 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "lodash": "^4.15.0", "nan": "^2.0.0", "node-pre-gyp": "^0.6.0", - "protobufjs": "^5.0.0" + "protobufjs": "^6.0.0" }, "devDependencies": { "async": "^2.0.1", diff --git a/src/node/index.js b/src/node/index.js index a294aad8ee6..43a4a54dd8d 100644 --- a/src/node/index.js +++ b/src/node/index.js @@ -52,32 +52,36 @@ var Metadata = require('./src/metadata.js'); var grpc = require('./src/grpc_extension'); +var protobuf_js_5_common = require('./src/protobuf_js_5_common'); +var protobuf_js_6_common = require('./src/protobuf_js_6_common'); + grpc.setDefaultRootsPem(fs.readFileSync(SSL_ROOTS_PATH, 'ascii')); -/** - * Load a gRPC object from an existing ProtoBuf.Reflect object. - * @param {ProtoBuf.Reflect.Namespace} value The ProtoBuf object to load. - * @param {Object=} options Options to apply to the loaded object - * @return {Object} The resulting gRPC object - */ exports.loadObject = function loadObject(value, options) { - var result = {}; - if (value.className === 'Namespace') { - _.each(value.children, function(child) { - result[child.name] = loadObject(child, options); - }); - return result; - } else if (value.className === 'Service') { - return client.makeProtobufClientConstructor(value, options); - } else if (value.className === 'Message' || value.className === 'Enum') { - return value.build(); - } else { - return value; + options = _.defaults(options, {'protobufjs_version': 5}); + switch (options.protobufjs_version) { + case 5: return protobuf_js_5_common.loadObject(value, options); + case 6: return protobuf_js_6_common.loadObject(value, options); + default: throw new Error('Unrecognized protobufjs_version:', + options.protobufjs_version); } }; var loadObject = exports.loadObject; +function applyProtoRoot(filename, root) { + if (_.isString(filename)) { + return filename; + } + filename.root = path.resolve(filename.root) + '/'; + root.resolvePath = function(originPath, importPath, alreadyNormalized) { + return ProtoBuf.util.path.resolve(filename.root, + importPath, + alreadyNormalized); + }; + return filename.file; +} + /** * Load a gRPC object from a .proto file. The options object can provide the * following options: @@ -99,29 +103,17 @@ var loadObject = exports.loadObject; * @return {Object} The resulting gRPC object */ exports.load = function load(filename, format, options) { - if (!format) { - format = 'proto'; - } - var convertFieldsToCamelCaseOriginal = ProtoBuf.convertFieldsToCamelCase; - if(options && options.hasOwnProperty('convertFieldsToCamelCase')) { - ProtoBuf.convertFieldsToCamelCase = options.convertFieldsToCamelCase; - } - var builder; - try { - switch(format) { - case 'proto': - builder = ProtoBuf.loadProtoFile(filename); - break; - case 'json': - builder = ProtoBuf.loadJsonFile(filename); - break; - default: - throw new Error('Unrecognized format "' + format + '"'); - } - } finally { - ProtoBuf.convertFieldsToCamelCase = convertFieldsToCamelCaseOriginal; - } - return loadObject(builder.ns, options); + /* Note: format is currently unused, because the API for loading a proto + file or a JSON file is identical in Protobuf.js 6. In the future, there is + still the possibility of adding other formats that would be loaded + differently */ + options = _.defaults(options, common.defaultGrpcOptions); + options.protobufjs_version = 6; + var root = new ProtoBuf.Root(); + var parse_options = {keepCase: !options.convertFieldsToCamelCase}; + return loadObject(root.loadSync(applyProtoRoot(filename, root), + parse_options), + options); }; var log_template = _.template( diff --git a/src/node/interop/interop_server.js b/src/node/interop/interop_server.js index 05f52a1083d..83b8a7c1ec3 100644 --- a/src/node/interop/interop_server.js +++ b/src/node/interop/interop_server.js @@ -228,7 +228,7 @@ function getServer(port, tls) { server_creds = grpc.ServerCredentials.createInsecure(); } var server = new grpc.Server(options); - server.addProtoService(testProto.TestService.service, { + server.addService(testProto.TestService.service, { emptyCall: handleEmpty, unaryCall: handleUnary, streamingOutputCall: handleStreamingOutput, diff --git a/src/node/performance/benchmark_server.js b/src/node/performance/benchmark_server.js index 6abde2e17af..297569164bb 100644 --- a/src/node/performance/benchmark_server.js +++ b/src/node/performance/benchmark_server.js @@ -132,7 +132,7 @@ function BenchmarkServer(host, port, tls, generic, response_size) { streamingCall: makeStreamingGenericCall(response_size) }); } else { - server.addProtoService(serviceProto.BenchmarkService.service, { + server.addService(serviceProto.BenchmarkService.service, { unaryCall: unaryCall, streamingCall: streamingCall }); diff --git a/src/node/performance/worker.js b/src/node/performance/worker.js index 030bf7d7ba0..90a9b7d59cd 100644 --- a/src/node/performance/worker.js +++ b/src/node/performance/worker.js @@ -44,8 +44,8 @@ var serviceProto = grpc.load({ function runServer(port, benchmark_impl) { var server_creds = grpc.ServerCredentials.createInsecure(); var server = new grpc.Server(); - server.addProtoService(serviceProto.WorkerService.service, - new WorkerServiceImpl(benchmark_impl, server)); + server.addService(serviceProto.WorkerService.service, + new WorkerServiceImpl(benchmark_impl, server)); var address = '0.0.0.0:' + port; server.bind(address, server_creds); server.start(); diff --git a/src/node/src/client.js b/src/node/src/client.js index 44081a3a6cd..1aaf35c16cb 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -780,6 +780,8 @@ exports.makeClientConstructor = function(methods, serviceName, _.assign(Client.prototype[name], attrs); }); + Client.service = methods; + return Client; }; @@ -822,26 +824,6 @@ exports.waitForClientReady = function(client, deadline, callback) { checkState(); }; -/** - * Creates a constructor for clients for the given service - * @param {ProtoBuf.Reflect.Service} service The service to generate a client - * for - * @param {Object=} options Options to apply to the client - * @return {function(string, Object)} New client constructor - */ -exports.makeProtobufClientConstructor = function(service, options) { - var method_attrs = common.getProtobufServiceAttrs(service, options); - if (!options) { - options = {deprecatedArgumentOrder: false}; - } - var Client = exports.makeClientConstructor( - method_attrs, common.fullyQualifiedName(service), - options); - Client.service = service; - Client.service.grpc_options = options; - return Client; -}; - /** * Map of status code names to status codes */ diff --git a/src/node/src/common.js b/src/node/src/common.js index 98eabf5c0b8..93df5138172 100644 --- a/src/node/src/common.js +++ b/src/node/src/common.js @@ -41,74 +41,6 @@ var _ = require('lodash'); -/** - * Get a function that deserializes a specific type of protobuf. - * @param {function()} cls The constructor of the message type to deserialize - * @param {bool=} binaryAsBase64 Deserialize bytes fields as base64 strings - * instead of Buffers. Defaults to false - * @param {bool=} longsAsStrings Deserialize long values as strings instead of - * objects. Defaults to true - * @return {function(Buffer):cls} The deserialization function - */ -exports.deserializeCls = function deserializeCls(cls, binaryAsBase64, - longsAsStrings) { - if (binaryAsBase64 === undefined || binaryAsBase64 === null) { - binaryAsBase64 = false; - } - if (longsAsStrings === undefined || longsAsStrings === null) { - longsAsStrings = true; - } - /** - * Deserialize a buffer to a message object - * @param {Buffer} arg_buf The buffer to deserialize - * @return {cls} The resulting object - */ - return function deserialize(arg_buf) { - // Convert to a native object with binary fields as Buffers (first argument) - // and longs as strings (second argument) - return cls.decode(arg_buf).toRaw(binaryAsBase64, longsAsStrings); - }; -}; - -var deserializeCls = exports.deserializeCls; - -/** - * Get a function that serializes objects to a buffer by protobuf class. - * @param {function()} Cls The constructor of the message type to serialize - * @return {function(Cls):Buffer} The serialization function - */ -exports.serializeCls = function serializeCls(Cls) { - /** - * Serialize an object to a Buffer - * @param {Object} arg The object to serialize - * @return {Buffer} The serialized object - */ - return function serialize(arg) { - return new Buffer(new Cls(arg).encode().toBuffer()); - }; -}; - -var serializeCls = exports.serializeCls; - -/** - * Get the fully qualified (dotted) name of a ProtoBuf.Reflect value. - * @param {ProtoBuf.Reflect.Namespace} value The value to get the name of - * @return {string} The fully qualified name of the value - */ -exports.fullyQualifiedName = function fullyQualifiedName(value) { - if (value === null || value === undefined) { - return ''; - } - var name = value.name; - var parent_name = fullyQualifiedName(value.parent); - if (parent_name !== '') { - name = parent_name + '.' + name; - } - return name; -}; - -var fullyQualifiedName = exports.fullyQualifiedName; - /** * Wrap a function to pass null-like values through without calling it. If no * function is given, just uses the identity; @@ -127,43 +59,6 @@ exports.wrapIgnoreNull = function wrapIgnoreNull(func) { }; }; -/** - * Return a map from method names to method attributes for the service. - * @param {ProtoBuf.Reflect.Service} service The service to get attributes for - * @param {Object=} options Options to apply to these attributes - * @return {Object} The attributes map - */ -exports.getProtobufServiceAttrs = function getProtobufServiceAttrs(service, - options) { - var prefix = '/' + fullyQualifiedName(service) + '/'; - var binaryAsBase64, longsAsStrings; - if (options) { - binaryAsBase64 = options.binaryAsBase64; - longsAsStrings = options.longsAsStrings; - } - /* This slightly awkward construction is used to make sure we only use - lodash@3.10.1-compatible functions. A previous version used - _.fromPairs, which would be cleaner, but was introduced in lodash - version 4 */ - return _.zipObject(_.map(service.children, function(method) { - return _.camelCase(method.name); - }), _.map(service.children, function(method) { - return { - path: prefix + method.name, - requestStream: method.requestStream, - responseStream: method.responseStream, - requestType: method.resolvedRequestType, - responseType: method.resolvedResponseType, - requestSerialize: serializeCls(method.resolvedRequestType.build()), - requestDeserialize: deserializeCls(method.resolvedRequestType.build(), - binaryAsBase64, longsAsStrings), - responseSerialize: serializeCls(method.resolvedResponseType.build()), - responseDeserialize: deserializeCls(method.resolvedResponseType.build(), - binaryAsBase64, longsAsStrings) - }; - })); -}; - /** * The logger object for the gRPC module. Defaults to console. */ @@ -184,3 +79,13 @@ exports.log = function log(severity, message) { exports.logger.error(message); } }; + +/** + * Default options for loading proto files into gRPC + */ +exports.defaultGrpcOptions = { + convertFieldsToCamelCase: false, + binaryAsBase64: false, + longsAsStrings: true, + deprecatedArgumentOrder: false +}; diff --git a/src/node/src/protobuf_js_5_common.js b/src/node/src/protobuf_js_5_common.js new file mode 100644 index 00000000000..aecb25339bf --- /dev/null +++ b/src/node/src/protobuf_js_5_common.js @@ -0,0 +1,167 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +'use strict'; + +var _ = require('lodash'); +var client = require('./client'); + +/** + * Get a function that deserializes a specific type of protobuf. + * @param {function()} cls The constructor of the message type to deserialize + * @param {bool=} binaryAsBase64 Deserialize bytes fields as base64 strings + * instead of Buffers. Defaults to false + * @param {bool=} longsAsStrings Deserialize long values as strings instead of + * objects. Defaults to true + * @return {function(Buffer):cls} The deserialization function + */ +exports.deserializeCls = function deserializeCls(cls, binaryAsBase64, + longsAsStrings) { + /** + * Deserialize a buffer to a message object + * @param {Buffer} arg_buf The buffer to deserialize + * @return {cls} The resulting object + */ + return function deserialize(arg_buf) { + // Convert to a native object with binary fields as Buffers (first argument) + // and longs as strings (second argument) + return cls.decode(arg_buf).toRaw(binaryAsBase64, longsAsStrings); + }; +}; + +var deserializeCls = exports.deserializeCls; + +/** + * Get a function that serializes objects to a buffer by protobuf class. + * @param {function()} Cls The constructor of the message type to serialize + * @return {function(Cls):Buffer} The serialization function + */ +exports.serializeCls = function serializeCls(Cls) { + /** + * Serialize an object to a Buffer + * @param {Object} arg The object to serialize + * @return {Buffer} The serialized object + */ + return function serialize(arg) { + return new Buffer(new Cls(arg).encode().toBuffer()); + }; +}; + +var serializeCls = exports.serializeCls; + +/** + * Get the fully qualified (dotted) name of a ProtoBuf.Reflect value. + * @param {ProtoBuf.Reflect.Namespace} value The value to get the name of + * @return {string} The fully qualified name of the value + */ +exports.fullyQualifiedName = function fullyQualifiedName(value) { + if (value === null || value === undefined) { + return ''; + } + var name = value.name; + var parent_name = fullyQualifiedName(value.parent); + if (parent_name !== '') { + name = parent_name + '.' + name; + } + return name; +}; + +var fullyQualifiedName = exports.fullyQualifiedName; + +/** + * Return a map from method names to method attributes for the service. + * @param {ProtoBuf.Reflect.Service} service The service to get attributes for + * @param {Object=} options Options to apply to these attributes + * @return {Object} The attributes map + */ +exports.getProtobufServiceAttrs = function getProtobufServiceAttrs(service, + options) { + var prefix = '/' + fullyQualifiedName(service) + '/'; + var binaryAsBase64, longsAsStrings; + if (options) { + binaryAsBase64 = options.binaryAsBase64; + longsAsStrings = options.longsAsStrings; + } + /* This slightly awkward construction is used to make sure we only use + lodash@3.10.1-compatible functions. A previous version used + _.fromPairs, which would be cleaner, but was introduced in lodash + version 4 */ + return _.zipObject(_.map(service.children, function(method) { + return _.camelCase(method.name); + }), _.map(service.children, function(method) { + return { + path: prefix + method.name, + requestStream: method.requestStream, + responseStream: method.responseStream, + requestType: method.resolvedRequestType, + responseType: method.resolvedResponseType, + requestSerialize: serializeCls(method.resolvedRequestType.build()), + requestDeserialize: deserializeCls(method.resolvedRequestType.build(), + binaryAsBase64, longsAsStrings), + responseSerialize: serializeCls(method.resolvedResponseType.build()), + responseDeserialize: deserializeCls(method.resolvedResponseType.build(), + binaryAsBase64, longsAsStrings) + }; + })); +}; + +var getProtobufServiceAttrs = exports.getProtobufServiceAttrs; + +/** + * Load a gRPC object from an existing ProtoBuf.Reflect object. + * @param {ProtoBuf.Reflect.Namespace} value The ProtoBuf object to load. + * @param {Object=} options Options to apply to the loaded object + * @return {Object} The resulting gRPC object + */ +exports.loadObject = function loadObject(value, options) { + var result = {}; + if (!value) { + return value; + } + if (value.hasOwnProperty('ns')) { + return loadObject(value.ns, options); + } + if (value.className === 'Namespace') { + _.each(value.children, function(child) { + result[child.name] = loadObject(child, options); + }); + return result; + } else if (value.className === 'Service') { + return client.makeClientConstructor(getProtobufServiceAttrs(value, options), + options); + } else if (value.className === 'Message' || value.className === 'Enum') { + return value.build(); + } else { + return value; + } +}; diff --git a/src/node/src/protobuf_js_6_common.js b/src/node/src/protobuf_js_6_common.js new file mode 100644 index 00000000000..23e56eecdeb --- /dev/null +++ b/src/node/src/protobuf_js_6_common.js @@ -0,0 +1,154 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +'use strict'; + +var _ = require('lodash'); +var client = require('./client'); + +/** + * Get a function that deserializes a specific type of protobuf. + * @param {function()} cls The constructor of the message type to deserialize + * @param {bool=} binaryAsBase64 Deserialize bytes fields as base64 strings + * instead of Buffers. Defaults to false + * @param {bool=} longsAsStrings Deserialize long values as strings instead of + * objects. Defaults to true + * @return {function(Buffer):cls} The deserialization function + */ +exports.deserializeCls = function deserializeCls(cls, options) { + var conversion_options = { + defaults: true, + bytes: options.binaryAsBase64 ? String : Buffer, + longs: options.longsAsStrings ? String : null, + enums: String + }; + /** + * Deserialize a buffer to a message object + * @param {Buffer} arg_buf The buffer to deserialize + * @return {cls} The resulting object + */ + return function deserialize(arg_buf) { + return cls.decode(arg_buf).toObject(conversion_options); + }; +}; + +var deserializeCls = exports.deserializeCls; + +/** + * Get a function that serializes objects to a buffer by protobuf class. + * @param {function()} Cls The constructor of the message type to serialize + * @return {function(Cls):Buffer} The serialization function + */ +exports.serializeCls = function serializeCls(cls) { + /** + * Serialize an object to a Buffer + * @param {Object} arg The object to serialize + * @return {Buffer} The serialized object + */ + return function serialize(arg) { + return cls.encode(arg).finish(); + }; +}; + +var serializeCls = exports.serializeCls; + +/** + * Get the fully qualified (dotted) name of a ProtoBuf.Reflect value. + * @param {ProtoBuf.ReflectionObject} value The value to get the name of + * @return {string} The fully qualified name of the value + */ +exports.fullyQualifiedName = function fullyQualifiedName(value) { + if (value === null || value === undefined) { + return ''; + } + var name = value.name; + var parent_fqn = fullyQualifiedName(value.parent); + if (parent_fqn !== '') { + name = parent_fqn + '.' + name; + } + return name; +}; + +var fullyQualifiedName = exports.fullyQualifiedName; + +/** + * Return a map from method names to method attributes for the service. + * @param {ProtoBuf.Service} service The service to get attributes for + * @param {Object=} options Options to apply to these attributes + * @return {Object} The attributes map + */ +exports.getProtobufServiceAttrs = function getProtobufServiceAttrs(service, + options) { + var prefix = '/' + fullyQualifiedName(service) + '/'; + service.resolveAll(); + return _.zipObject(_.map(service.methods, function(method) { + return _.camelCase(method.name); + }), _.map(service.methods, function(method) { + return { + path: prefix + method.name, + requestStream: !!method.requestStream, + responseStream: !!method.responseStream, + requestType: method.resolvedRequestType, + responseType: method.resolvedResponseType, + requestSerialize: serializeCls(method.resolvedRequestType), + requestDeserialize: deserializeCls(method.resolvedRequestType, options), + responseSerialize: serializeCls(method.resolvedResponseType), + responseDeserialize: deserializeCls(method.resolvedResponseType, options) + }; + })); +}; + +var getProtobufServiceAttrs = exports.getProtobufServiceAttrs; + +exports.loadObject = function loadObject(value, options) { + var result = {}; + if (!value) { + return value; + } + if (value.hasOwnProperty('methods')) { + // It's a service object + var service_attrs = getProtobufServiceAttrs(value, options); + return client.makeClientConstructor(service_attrs); + } + + if (value.hasOwnProperty('nested')) { + // It's a namespace or root object + _.each(value.nested, function(nested, name) { + result[name] = loadObject(nested, options); + }); + return result; + } + + // Otherwise, it's not something we need to change + return value; +}; diff --git a/src/node/src/server.js b/src/node/src/server.js index a5a0ea26420..9a149ebf0a5 100644 --- a/src/node/src/server.js +++ b/src/node/src/server.js @@ -774,17 +774,33 @@ Server.prototype.addService = function(service, implementation) { /** * Add a proto service to the server, with a corresponding implementation + * @deprecated Use grpc.load and Server#addService instead * @param {Protobuf.Reflect.Service} service The proto service descriptor * @param {Object} implementation Map of method names to * method implementation for the provided service. */ Server.prototype.addProtoService = function(service, implementation) { var options; - if (service.grpc_options) { - options = service.grpc_options; + common.log(grpc.logVerbosity.INFO, + 'Server#addProtoService is deprecated. Use addService instead'); + if (service.hasOwnProperty('children')) { + // Heuristically: this is a Protobuf.js 5 service object + var protobuf_js_5_common = require('./protobuf_js_5_common'); + options = _.defaults(service.grpc_options, common.defaultGrpcOptions); + this.addService( + protobuf_js_5_common.getProtobufServiceAttrs(service, options), + implementation); + } else if (service.hasOwnProperty('methods')) { + // Heuristically: this is a Protobuf.js 6 service object + var protobuf_js_6_common = require('./protobuf_js_6_common'); + options = _.defaults(service.grpc_options, common.defaultGrpcOptions); + this.addService( + protobuf_js_6_common.getProtobufServiceAttrs(service, options), + implementation); + } else { + // We assume that this is a service attributes object + this.addService(service, implementation); } - this.addService(common.getProtobufServiceAttrs(service, options), - implementation); }; /** diff --git a/src/node/stress/metrics_server.js b/src/node/stress/metrics_server.js index 3ab4b4c82d3..b3f939e8f3d 100644 --- a/src/node/stress/metrics_server.js +++ b/src/node/stress/metrics_server.js @@ -63,7 +63,7 @@ function getAllGauges(call) { function MetricsServer(port) { var server = new grpc.Server(); - server.addProtoService(metrics.MetricsService.service, { + server.addService(metrics.MetricsService.service, { getGauge: _.bind(getGauge, this), getAllGauges: _.bind(getAllGauges, this) }); diff --git a/src/node/test/common_test.js b/src/node/test/common_test.js index c57b7388f67..529e82e3242 100644 --- a/src/node/test/common_test.js +++ b/src/node/test/common_test.js @@ -34,17 +34,26 @@ 'use strict'; var assert = require('assert'); +var _ = require('lodash'); -var common = require('../src/common.js'); +var common = require('../src/common'); +var protobuf_js_6_common = require('../src/protobuf_js_6_common'); + +var serializeCls = protobuf_js_6_common.serializeCls; +var deserializeCls = protobuf_js_6_common.deserializeCls; var ProtoBuf = require('protobufjs'); -var messages_proto = ProtoBuf.loadProtoFile( - __dirname + '/test_messages.proto').build(); +var messages_proto = new ProtoBuf.Root(); +messages_proto = messages_proto.loadSync( + __dirname + '/test_messages.proto', {keepCase: true}).resolveAll(); + +var default_options = common.defaultGrpcOptions; describe('Proto message long int serialize and deserialize', function() { - var longSerialize = common.serializeCls(messages_proto.LongValues); - var longDeserialize = common.deserializeCls(messages_proto.LongValues); + var longSerialize = serializeCls(messages_proto.LongValues); + var longDeserialize = deserializeCls(messages_proto.LongValues, + default_options); var pos_value = '314159265358979'; var neg_value = '-27182818284590'; it('should preserve positive int64 values', function() { @@ -88,8 +97,9 @@ describe('Proto message long int serialize and deserialize', function() { neg_value); }); it('should deserialize as a number with the right option set', function() { - var longNumDeserialize = common.deserializeCls(messages_proto.LongValues, - false, false); + var num_options = _.defaults({longsAsStrings: false}, default_options); + var longNumDeserialize = deserializeCls(messages_proto.LongValues, + num_options); var serialized = longSerialize({int_64: pos_value}); assert.strictEqual(typeof longDeserialize(serialized).int_64, 'string'); /* With the longsAsStrings option disabled, long values are represented as @@ -98,11 +108,12 @@ describe('Proto message long int serialize and deserialize', function() { }); }); describe('Proto message bytes serialize and deserialize', function() { - var sequenceSerialize = common.serializeCls(messages_proto.SequenceValues); - var sequenceDeserialize = common.deserializeCls( - messages_proto.SequenceValues); - var sequenceBase64Deserialize = common.deserializeCls( - messages_proto.SequenceValues, true); + var sequenceSerialize = serializeCls(messages_proto.SequenceValues); + var sequenceDeserialize = deserializeCls( + messages_proto.SequenceValues, default_options); + var b64_options = _.defaults({binaryAsBase64: true}, default_options); + var sequenceBase64Deserialize = deserializeCls( + messages_proto.SequenceValues, b64_options); var buffer_val = new Buffer([0x69, 0xb7]); var base64_val = 'abc='; it('should preserve a buffer', function() { diff --git a/src/node/test/credentials_test.js b/src/node/test/credentials_test.js index 305843f665e..b66b4bf5ea4 100644 --- a/src/node/test/credentials_test.js +++ b/src/node/test/credentials_test.js @@ -228,7 +228,7 @@ describe('client credentials', function() { before(function() { var proto = grpc.load(__dirname + '/test_service.proto'); server = new grpc.Server(); - server.addProtoService(proto.TestService.service, { + server.addService(proto.TestService.service, { unary: function(call, cb) { call.sendMetadata(call.metadata); cb(null, {}); diff --git a/src/node/test/surface_test.js b/src/node/test/surface_test.js index 2636ea85ac8..1e3fe8fb575 100644 --- a/src/node/test/surface_test.js +++ b/src/node/test/surface_test.js @@ -34,19 +34,23 @@ 'use strict'; var assert = require('assert'); +var _ = require('lodash'); var surface_client = require('../src/client.js'); +var common = require('../src/common'); var ProtoBuf = require('protobufjs'); var grpc = require('..'); -var math_proto = ProtoBuf.loadProtoFile(__dirname + - '/../../proto/math/math.proto'); +var math_proto = new ProtoBuf.Root(); +math_proto = math_proto.loadSync(__dirname + + '/../../proto/math/math.proto', {keepCase: true}); var mathService = math_proto.lookup('math.Math'); - -var _ = require('lodash'); +var mathServiceAttrs = grpc.loadObject( + mathService, + _.defaults({protobufjs_version: 6}, common.defaultGrpcOptions)).service; /** * This is used for testing functions with multiple asynchronous calls that @@ -87,11 +91,6 @@ describe('File loader', function() { grpc.load(__dirname + '/test_service.json', 'json'); }); }); - it('Should fail to load a file with an unknown format', function() { - assert.throws(function() { - grpc.load(__dirname + '/test_service.proto', 'fake_format'); - }); - }); }); describe('surface Server', function() { var server; @@ -132,29 +131,54 @@ describe('Server.prototype.addProtoService', function() { afterEach(function() { server.forceShutdown(); }); - it('Should succeed with a single service', function() { + it('Should succeed with a single proto service', function() { assert.doesNotThrow(function() { server.addProtoService(mathService, dummyImpls); }); }); + it('Should succeed with a single service attributes object', function() { + assert.doesNotThrow(function() { + server.addProtoService(mathServiceAttrs, dummyImpls); + }); + }); +}); +describe('Server.prototype.addService', function() { + var server; + var dummyImpls = { + 'div': function() {}, + 'divMany': function() {}, + 'fib': function() {}, + 'sum': function() {} + }; + beforeEach(function() { + server = new grpc.Server(); + }); + afterEach(function() { + server.forceShutdown(); + }); + it('Should succeed with a single service', function() { + assert.doesNotThrow(function() { + server.addService(mathServiceAttrs, dummyImpls); + }); + }); it('Should fail with conflicting method names', function() { - server.addProtoService(mathService, dummyImpls); + server.addService(mathServiceAttrs, dummyImpls); assert.throws(function() { - server.addProtoService(mathService, dummyImpls); + server.addService(mathServiceAttrs, dummyImpls); }); }); it('Should fail if the server has been started', function() { server.start(); assert.throws(function() { - server.addProtoService(mathService, dummyImpls); + server.addService(mathServiceAttrs, dummyImpls); }); }); describe('Default handlers', function() { var client; beforeEach(function() { - server.addProtoService(mathService, {}); + server.addService(mathServiceAttrs, {}); var port = server.bind('localhost:0', server_insecure_creds); - var Client = surface_client.makeProtobufClientConstructor(mathService); + var Client = grpc.loadObject(mathService, {protobufjs_version: 6}); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); @@ -226,7 +250,7 @@ describe('waitForClientReady', function() { server = new grpc.Server(); port = server.bind('localhost:0', grpc.ServerCredentials.createInsecure()); server.start(); - Client = surface_client.makeProtobufClientConstructor(mathService); + Client = grpc.loadObject(mathService, {protobufjs_version: 6}); }); beforeEach(function() { client = new Client('localhost:' + port, grpc.credentials.createInsecure()); @@ -283,16 +307,18 @@ describe('Echo service', function() { var server; var client; before(function() { - var test_proto = ProtoBuf.loadProtoFile(__dirname + '/echo_service.proto'); + var test_proto = new ProtoBuf.Root(); + test_proto = test_proto.loadSync(__dirname + '/echo_service.proto', + {keepCase: true}); var echo_service = test_proto.lookup('EchoService'); + var Client = grpc.loadObject(echo_service, {protobufjs_version: 6}); server = new grpc.Server(); - server.addProtoService(echo_service, { + server.addService(Client.service, { echo: function(call, callback) { callback(null, call.request); } }); var port = server.bind('localhost:0', server_insecure_creds); - var Client = surface_client.makeProtobufClientConstructor(echo_service); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); }); @@ -406,10 +432,13 @@ describe('Echo metadata', function() { var server; var metadata; before(function() { - var test_proto = ProtoBuf.loadProtoFile(__dirname + '/test_service.proto'); + var test_proto = new ProtoBuf.Root(); + test_proto = test_proto.loadSync(__dirname + '/test_service.proto', + {keepCase: true}); var test_service = test_proto.lookup('TestService'); + var Client = grpc.loadObject(test_service, {protobufjs_version: 6}); server = new grpc.Server(); - server.addProtoService(test_service, { + server.addService(Client.service, { unary: function(call, cb) { call.sendMetadata(call.metadata); cb(null, {}); @@ -434,7 +463,6 @@ describe('Echo metadata', function() { } }); var port = server.bind('localhost:0', server_insecure_creds); - var Client = surface_client.makeProtobufClientConstructor(test_service); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); metadata = new grpc.Metadata(); @@ -507,7 +535,9 @@ describe('Client malformed response handling', function() { var client; var badArg = new Buffer([0xFF]); before(function() { - var test_proto = ProtoBuf.loadProtoFile(__dirname + '/test_service.proto'); + var test_proto = new ProtoBuf.Root(); + test_proto = test_proto.loadSync(__dirname + '/test_service.proto', + {keepCase: true}); var test_service = test_proto.lookup('TestService'); var malformed_test_service = { unary: { @@ -565,7 +595,7 @@ describe('Client malformed response handling', function() { } }); var port = server.bind('localhost:0', server_insecure_creds); - var Client = surface_client.makeProtobufClientConstructor(test_service); + var Client = grpc.loadObject(test_service, {protobufjs_version: 6}); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); }); @@ -614,7 +644,9 @@ describe('Server serialization failure handling', function() { var client; var server; before(function() { - var test_proto = ProtoBuf.loadProtoFile(__dirname + '/test_service.proto'); + var test_proto = new ProtoBuf.Root(); + test_proto = test_proto.loadSync(__dirname + '/test_service.proto', + {keepCase: true}); var test_service = test_proto.lookup('TestService'); var malformed_test_service = { unary: { @@ -672,7 +704,7 @@ describe('Server serialization failure handling', function() { } }); var port = server.bind('localhost:0', server_insecure_creds); - var Client = surface_client.makeProtobufClientConstructor(test_service); + var Client = grpc.loadObject(test_service, {protobufjs_version: 6}); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); }); @@ -721,12 +753,15 @@ describe('Other conditions', function() { var server; var port; before(function() { - var test_proto = ProtoBuf.loadProtoFile(__dirname + '/test_service.proto'); + var test_proto = new ProtoBuf.Root(); + test_proto = test_proto.loadSync(__dirname + '/test_service.proto', + {keepCase: true}); test_service = test_proto.lookup('TestService'); + Client = grpc.loadObject(test_service, {protobufjs_version: 6}); server = new grpc.Server(); var trailer_metadata = new grpc.Metadata(); trailer_metadata.add('trailer-present', 'yes'); - server.addProtoService(test_service, { + server.addService(Client.service, { unary: function(call, cb) { var req = call.request; if (req.error) { @@ -786,7 +821,6 @@ describe('Other conditions', function() { } }); port = server.bind('localhost:0', server_insecure_creds); - Client = surface_client.makeProtobufClientConstructor(test_service); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); }); @@ -1067,17 +1101,19 @@ describe('Call propagation', function() { var client; var server; before(function() { - var test_proto = ProtoBuf.loadProtoFile(__dirname + '/test_service.proto'); + var test_proto = new ProtoBuf.Root(); + test_proto = test_proto.loadSync(__dirname + '/test_service.proto', + {keepCase: true}); test_service = test_proto.lookup('TestService'); server = new grpc.Server(); - server.addProtoService(test_service, { + Client = grpc.loadObject(test_service, {protobufjs_version: 6}); + server.addService(Client.service, { unary: function(call) {}, clientStream: function(stream) {}, serverStream: function(stream) {}, bidiStream: function(stream) {} }); var port = server.bind('localhost:0', server_insecure_creds); - Client = surface_client.makeProtobufClientConstructor(test_service); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); }); @@ -1112,7 +1148,7 @@ describe('Call propagation', function() { }); call.cancel(); }; - proxy.addProtoService(test_service, proxy_impl); + proxy.addService(Client.service, proxy_impl); var proxy_port = proxy.bind('localhost:0', server_insecure_creds); proxy.start(); var proxy_client = new Client('localhost:' + proxy_port, @@ -1134,7 +1170,7 @@ describe('Call propagation', function() { }); call.cancel(); }; - proxy.addProtoService(test_service, proxy_impl); + proxy.addService(Client.service, proxy_impl); var proxy_port = proxy.bind('localhost:0', server_insecure_creds); proxy.start(); var proxy_client = new Client('localhost:' + proxy_port, @@ -1154,7 +1190,7 @@ describe('Call propagation', function() { }); call.cancel(); }; - proxy.addProtoService(test_service, proxy_impl); + proxy.addService(Client.service, proxy_impl); var proxy_port = proxy.bind('localhost:0', server_insecure_creds); proxy.start(); var proxy_client = new Client('localhost:' + proxy_port, @@ -1178,7 +1214,7 @@ describe('Call propagation', function() { }); call.cancel(); }; - proxy.addProtoService(test_service, proxy_impl); + proxy.addService(Client.service, proxy_impl); var proxy_port = proxy.bind('localhost:0', server_insecure_creds); proxy.start(); var proxy_client = new Client('localhost:' + proxy_port, @@ -1209,7 +1245,7 @@ describe('Call propagation', function() { } }); }; - proxy.addProtoService(test_service, proxy_impl); + proxy.addService(Client.service, proxy_impl); var proxy_port = proxy.bind('localhost:0', server_insecure_creds); proxy.start(); var proxy_client = new Client('localhost:' + proxy_port, @@ -1233,7 +1269,7 @@ describe('Call propagation', function() { done(); }); }; - proxy.addProtoService(test_service, proxy_impl); + proxy.addService(Client.service, proxy_impl); var proxy_port = proxy.bind('localhost:0', server_insecure_creds); proxy.start(); var proxy_client = new Client('localhost:' + proxy_port, @@ -1253,14 +1289,14 @@ describe('Cancelling surface client', function() { var server; before(function() { server = new grpc.Server(); - server.addProtoService(mathService, { + server.addService(mathServiceAttrs, { 'div': function(stream) {}, 'divMany': function(stream) {}, 'fib': function(stream) {}, 'sum': function(stream) {} }); var port = server.bind('localhost:0', server_insecure_creds); - var Client = surface_client.makeProtobufClientConstructor(mathService); + var Client = surface_client.makeClientConstructor(mathServiceAttrs); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); }); diff --git a/templates/package.json.template b/templates/package.json.template index 316c28e478e..e039408e331 100644 --- a/templates/package.json.template +++ b/templates/package.json.template @@ -36,7 +36,7 @@ "lodash": "^4.15.0", "nan": "^2.0.0", "node-pre-gyp": "^0.6.0", - "protobufjs": "^5.0.0" + "protobufjs": "^6.0.0" }, "devDependencies": { "async": "^2.0.1", From b1a02311b37c35b8813386d2d7fbb02e1e61d616 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 17 Mar 2017 13:45:15 -0700 Subject: [PATCH 07/90] Autodetect ProtoBuf.js version in grpc.loadObject --- src/node/index.js | 18 +++++++++++------- src/node/test/surface_test.js | 19 +++++++++---------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/node/index.js b/src/node/index.js index 43a4a54dd8d..e41664ebf38 100644 --- a/src/node/index.js +++ b/src/node/index.js @@ -58,12 +58,17 @@ var protobuf_js_6_common = require('./src/protobuf_js_6_common'); grpc.setDefaultRootsPem(fs.readFileSync(SSL_ROOTS_PATH, 'ascii')); exports.loadObject = function loadObject(value, options) { - options = _.defaults(options, {'protobufjs_version': 5}); - switch (options.protobufjs_version) { - case 5: return protobuf_js_5_common.loadObject(value, options); - case 6: return protobuf_js_6_common.loadObject(value, options); - default: throw new Error('Unrecognized protobufjs_version:', - options.protobufjs_version); + options = _.defaults(options, common.defaultGrpcOptions); + if (value instanceof ProtoBuf.ReflectionObject) { + return protobuf_js_6_common.loadObject(value, options); + } else { + /* If value is not a ProtoBuf.js 6 reflection object, we assume that it is + a ProtoBuf.js 5 reflection object, for backwards compatibility */ + var deprecation_message = 'Calling grpc.loadObject with an object ' + + 'generated by ProtoBuf.js 5 is deprecated. Please upgrade to ' + + 'ProtoBuf.js 6.'; + common.log(grpc.logVerbosity.INFO, deprecation_message); + return protobuf_js_5_common.loadObject(value, options); } }; @@ -108,7 +113,6 @@ exports.load = function load(filename, format, options) { still the possibility of adding other formats that would be loaded differently */ options = _.defaults(options, common.defaultGrpcOptions); - options.protobufjs_version = 6; var root = new ProtoBuf.Root(); var parse_options = {keepCase: !options.convertFieldsToCamelCase}; return loadObject(root.loadSync(applyProtoRoot(filename, root), diff --git a/src/node/test/surface_test.js b/src/node/test/surface_test.js index 1e3fe8fb575..026d2c7dc25 100644 --- a/src/node/test/surface_test.js +++ b/src/node/test/surface_test.js @@ -49,8 +49,7 @@ math_proto = math_proto.loadSync(__dirname + var mathService = math_proto.lookup('math.Math'); var mathServiceAttrs = grpc.loadObject( - mathService, - _.defaults({protobufjs_version: 6}, common.defaultGrpcOptions)).service; + mathService, common.defaultGrpcOptions).service; /** * This is used for testing functions with multiple asynchronous calls that @@ -178,7 +177,7 @@ describe('Server.prototype.addService', function() { beforeEach(function() { server.addService(mathServiceAttrs, {}); var port = server.bind('localhost:0', server_insecure_creds); - var Client = grpc.loadObject(mathService, {protobufjs_version: 6}); + var Client = grpc.loadObject(mathService); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); @@ -250,7 +249,7 @@ describe('waitForClientReady', function() { server = new grpc.Server(); port = server.bind('localhost:0', grpc.ServerCredentials.createInsecure()); server.start(); - Client = grpc.loadObject(mathService, {protobufjs_version: 6}); + Client = grpc.loadObject(mathService); }); beforeEach(function() { client = new Client('localhost:' + port, grpc.credentials.createInsecure()); @@ -311,7 +310,7 @@ describe('Echo service', function() { test_proto = test_proto.loadSync(__dirname + '/echo_service.proto', {keepCase: true}); var echo_service = test_proto.lookup('EchoService'); - var Client = grpc.loadObject(echo_service, {protobufjs_version: 6}); + var Client = grpc.loadObject(echo_service); server = new grpc.Server(); server.addService(Client.service, { echo: function(call, callback) { @@ -436,7 +435,7 @@ describe('Echo metadata', function() { test_proto = test_proto.loadSync(__dirname + '/test_service.proto', {keepCase: true}); var test_service = test_proto.lookup('TestService'); - var Client = grpc.loadObject(test_service, {protobufjs_version: 6}); + var Client = grpc.loadObject(test_service); server = new grpc.Server(); server.addService(Client.service, { unary: function(call, cb) { @@ -595,7 +594,7 @@ describe('Client malformed response handling', function() { } }); var port = server.bind('localhost:0', server_insecure_creds); - var Client = grpc.loadObject(test_service, {protobufjs_version: 6}); + var Client = grpc.loadObject(test_service); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); }); @@ -704,7 +703,7 @@ describe('Server serialization failure handling', function() { } }); var port = server.bind('localhost:0', server_insecure_creds); - var Client = grpc.loadObject(test_service, {protobufjs_version: 6}); + var Client = grpc.loadObject(test_service); client = new Client('localhost:' + port, grpc.credentials.createInsecure()); server.start(); }); @@ -757,7 +756,7 @@ describe('Other conditions', function() { test_proto = test_proto.loadSync(__dirname + '/test_service.proto', {keepCase: true}); test_service = test_proto.lookup('TestService'); - Client = grpc.loadObject(test_service, {protobufjs_version: 6}); + Client = grpc.loadObject(test_service); server = new grpc.Server(); var trailer_metadata = new grpc.Metadata(); trailer_metadata.add('trailer-present', 'yes'); @@ -1106,7 +1105,7 @@ describe('Call propagation', function() { {keepCase: true}); test_service = test_proto.lookup('TestService'); server = new grpc.Server(); - Client = grpc.loadObject(test_service, {protobufjs_version: 6}); + Client = grpc.loadObject(test_service); server.addService(Client.service, { unary: function(call) {}, clientStream: function(stream) {}, From 2a63e1b28397d998aca50250990c5e5eef00f561 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 1 Mar 2017 18:48:43 -0800 Subject: [PATCH 08/90] Fix a bug in slice_buffer:maybe_embiggen --- src/core/lib/slice/slice_buffer.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/core/lib/slice/slice_buffer.c b/src/core/lib/slice/slice_buffer.c index 9176dc8a420..451319c50d6 100644 --- a/src/core/lib/slice/slice_buffer.c +++ b/src/core/lib/slice/slice_buffer.c @@ -46,11 +46,6 @@ #define GROW(x) (3 * (x) / 2) static void maybe_embiggen(grpc_slice_buffer *sb) { - if (sb->base_slices != sb->slices) { - memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice)); - sb->slices = sb->base_slices; - } - /* How far away from sb->base_slices is sb->slices pointer */ size_t slice_offset = (size_t)(sb->slices - sb->base_slices); size_t slice_count = sb->count + slice_offset; @@ -61,12 +56,15 @@ static void maybe_embiggen(grpc_slice_buffer *sb) { if (sb->base_slices == sb->inlined) { sb->base_slices = gpr_malloc(sb->capacity * sizeof(grpc_slice)); memcpy(sb->base_slices, sb->inlined, slice_count * sizeof(grpc_slice)); + sb->slices = sb->base_slices + slice_offset; } else { + if (sb->base_slices != sb->slices) { + memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice)); + } sb->base_slices = gpr_realloc(sb->base_slices, sb->capacity * sizeof(grpc_slice)); + sb->slices = sb->base_slices; } - - sb->slices = sb->base_slices + slice_offset; } } From c8e6ac4bf32bee5b22c04541a03ab11517a51651 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 20 Mar 2017 16:50:17 -0700 Subject: [PATCH 09/90] Ensure arguments are validated before they are serialized, unskip some tests --- src/node/src/protobuf_js_6_common.js | 7 ++++++- src/node/test/common_test.js | 26 +++++++++++++++++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/node/src/protobuf_js_6_common.js b/src/node/src/protobuf_js_6_common.js index 23e56eecdeb..ddee0ea9ee3 100644 --- a/src/node/src/protobuf_js_6_common.js +++ b/src/node/src/protobuf_js_6_common.js @@ -76,7 +76,12 @@ exports.serializeCls = function serializeCls(cls) { * @return {Buffer} The serialized object */ return function serialize(arg) { - return cls.encode(arg).finish(); + var message = cls.fromObject(arg); + var errMsg = cls.verify(message); + if (errMsg) { + throw errMsg; + } + return cls.encode(message).finish(); }; }; diff --git a/src/node/test/common_test.js b/src/node/test/common_test.js index 529e82e3242..f9efd8231d8 100644 --- a/src/node/test/common_test.js +++ b/src/node/test/common_test.js @@ -131,19 +131,27 @@ describe('Proto message bytes serialize and deserialize', function() { var deserialized = sequenceBase64Deserialize(serialized); assert.strictEqual(deserialized.bytes_field, base64_val); }); - /* The next two tests are specific tests to verify that issue - * https://github.com/grpc/grpc/issues/5174 has been fixed. They are skipped - * because they will not pass until a protobuf.js release has been published - * with a fix for https://github.com/dcodeIO/protobuf.js/issues/390 */ - it.skip('should serialize a repeated field as packed by default', function() { - var expected_serialize = new Buffer([0x12, 0x01, 0x01, 0x0a]); + it('should serialize a repeated field as packed by default', function() { + var expected_serialize = new Buffer([0x12, 0x01, 0x0a]); var serialized = sequenceSerialize({repeated_field: [10]}); assert.strictEqual(expected_serialize.compare(serialized), 0); }); - it.skip('should deserialize packed or unpacked repeated', function() { - var serialized = new Buffer([0x12, 0x01, 0x01, 0x0a]); + it('should deserialize packed or unpacked repeated', function() { + var expectedDeserialize = { + bytes_field: new Buffer(''), + repeated_field: [10] + }; + var packedSerialized = new Buffer([0x12, 0x01, 0x0a]); + var unpackedSerialized = new Buffer([0x10, 0x0a]); + var packedDeserialized; + var unpackedDeserialized; assert.doesNotThrow(function() { - sequenceDeserialize(serialized); + packedDeserialized = sequenceDeserialize(packedSerialized); }); + assert.doesNotThrow(function() { + unpackedDeserialized = sequenceDeserialize(unpackedSerialized); + }); + assert.deepEqual(packedDeserialized, expectedDeserialize); + assert.deepEqual(unpackedDeserialized, expectedDeserialize); }); }); From e8a44e064ab716424567b07b69b3b7acb744385c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 20 Mar 2017 17:34:09 -0700 Subject: [PATCH 10/90] Dynamically adjust TCP read buffer size --- src/core/lib/iomgr/tcp_posix.c | 47 +++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index a4381f8fc96..9fba8b3d0e2 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -52,6 +52,7 @@ #include #include #include +#include #include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/ev_posix.h" @@ -80,7 +81,8 @@ typedef struct { int fd; bool finished_edge; msg_iovlen_type iov_size; /* Number of slices to allocate per read attempt */ - size_t slice_size; + double target_length; + double bytes_read_this_round; gpr_refcount refcount; gpr_atm shutdown_count; @@ -108,6 +110,29 @@ typedef struct { grpc_resource_user_slice_allocator slice_allocator; } grpc_tcp; +static void add_to_estimate(grpc_tcp *tcp, size_t bytes) { + tcp->bytes_read_this_round += bytes; +} + +static void finish_estimate(grpc_tcp *tcp) { + if (tcp->bytes_read_this_round > tcp->target_length) { + tcp->target_length = 1.5 * tcp->bytes_read_this_round; + } else { + tcp->target_length = + 0.99 * tcp->target_length + 0.01 * tcp->bytes_read_this_round; + } + tcp->bytes_read_this_round = 0; +} + +static size_t get_target_read_size(grpc_tcp *tcp) { + double pressure = grpc_resource_quota_get_memory_pressure( + grpc_resource_user_quota(tcp->resource_user)); + double target = + tcp->target_length * (pressure > 0.8 ? (1.0 - pressure) / 0.2 : 1.0); + return (((size_t)GPR_CLAMP(target, 1024, 4 * 1024 * 1024)) + 255) & + ~(size_t)255; +} + static grpc_error *tcp_annotate_error(grpc_error *src_error, grpc_tcp *tcp) { return grpc_error_set_str( grpc_error_set_int(src_error, GRPC_ERROR_INT_FD, tcp->fd), @@ -231,9 +256,7 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { /* NB: After calling call_read_cb a parallel call of the read handler may * be running. */ if (errno == EAGAIN) { - if (tcp->iov_size > 1) { - tcp->iov_size /= 2; - } + finish_estimate(tcp); /* We've consumed the edge, request a new one */ grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure); } else { @@ -250,14 +273,13 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { tcp_annotate_error(GRPC_ERROR_CREATE("Socket closed"), tcp)); TCP_UNREF(exec_ctx, tcp, "read"); } else { + add_to_estimate(tcp, (size_t)read_bytes); GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length); if ((size_t)read_bytes < tcp->incoming_buffer->length) { grpc_slice_buffer_trim_end( tcp->incoming_buffer, tcp->incoming_buffer->length - (size_t)read_bytes, &tcp->last_read_buffer); - } else if (tcp->iov_size < MAX_READ_IOVEC) { - ++tcp->iov_size; } GPR_ASSERT((size_t)read_bytes == tcp->incoming_buffer->length); call_read_cb(exec_ctx, tcp, GRPC_ERROR_NONE); @@ -282,11 +304,11 @@ static void tcp_read_allocation_done(grpc_exec_ctx *exec_ctx, void *tcpp, } static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { - if (tcp->incoming_buffer->count < (size_t)tcp->iov_size) { - grpc_resource_user_alloc_slices( - exec_ctx, &tcp->slice_allocator, tcp->slice_size, - (size_t)tcp->iov_size - tcp->incoming_buffer->count, - tcp->incoming_buffer); + size_t target_read_size = get_target_read_size(tcp); + if (tcp->incoming_buffer->length < target_read_size && + tcp->incoming_buffer->count < MAX_READ_IOVEC) { + grpc_resource_user_alloc_slices(exec_ctx, &tcp->slice_allocator, + target_read_size, 1, tcp->incoming_buffer); } else { tcp_do_read(exec_ctx, tcp); } @@ -547,7 +569,8 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, tcp->release_fd_cb = NULL; tcp->release_fd = NULL; tcp->incoming_buffer = NULL; - tcp->slice_size = slice_size; + tcp->target_length = slice_size; + tcp->bytes_read_this_round = 0; tcp->iov_size = 1; tcp->finished_edge = true; /* paired with unref in grpc_tcp_destroy */ From b30685481679ca8a98328bbb98d872b577c7c39b Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 21 Mar 2017 11:25:01 -0700 Subject: [PATCH 11/90] Improve ProtoBuf.js version heuristic, add tests for oneof decoding --- src/node/index.js | 49 ++++++++++++++++++++++++---- src/node/src/protobuf_js_5_common.js | 13 ++++++++ src/node/src/protobuf_js_6_common.js | 13 ++++++++ src/node/src/server.js | 10 +++--- src/node/test/common_test.js | 24 ++++++++++++++ src/node/test/test_messages.proto | 7 ++++ 6 files changed, 103 insertions(+), 13 deletions(-) diff --git a/src/node/index.js b/src/node/index.js index e41664ebf38..ba73e3dac8f 100644 --- a/src/node/index.js +++ b/src/node/index.js @@ -57,18 +57,53 @@ var protobuf_js_6_common = require('./src/protobuf_js_6_common'); grpc.setDefaultRootsPem(fs.readFileSync(SSL_ROOTS_PATH, 'ascii')); +/** + * Load a ProtoBuf.js object as a gRPC object. The options object can provide + * the following options: + * - binaryAsBase64: deserialize bytes values as base64 strings instead of + * Buffers. Defaults to false + * - longsAsStrings: deserialize long values as strings instead of objects. + * Defaults to true + * - deprecatedArgumentOrder: Use the beta method argument order for client + * methods, with optional arguments after the callback. Defaults to false. + * This option is only a temporary stopgap measure to smooth an API breakage. + * It is deprecated, and new code should not use it. + * - protobufjsVersion: Available values are 5, 6, and 'detect'. 5 and 6 + * respectively indicate that an object from the corresponding version of + * ProtoBuf.js is provided in the value argument. If the option is 'detect', + * gRPC will guess what the version is based on the structure of the value. + * Defaults to 'detect'. + * @param {Object} value The ProtoBuf.js reflection object to load + * @param {Object=} options Options to apply to the loaded file + * @return {Object} The resulting gRPC object + */ exports.loadObject = function loadObject(value, options) { options = _.defaults(options, common.defaultGrpcOptions); - if (value instanceof ProtoBuf.ReflectionObject) { - return protobuf_js_6_common.loadObject(value, options); + options = _.defaults(options, {'protobufjsVersion': 'detect'}); + var protobufjsVersion; + if (options.protobufjsVersion === 'detect') { + if (protobuf_js_6_common.isProbablyProtobufJs6(value)) { + protobufjsVersion = 6; + } else if (protobuf_js_5_common.isProbablyProtobufJs5(value)) { + protobufjsVersion = 5; + } else { + var error_message = 'Could not detect ProtoBuf.js version. Please ' + + 'specify the version number with the "protobufjs_version" option'; + throw new Error(error_message); + } } else { - /* If value is not a ProtoBuf.js 6 reflection object, we assume that it is - a ProtoBuf.js 5 reflection object, for backwards compatibility */ + protobufjsVersion = options.protobufjsVersion; + } + switch (protobufjsVersion) { + case 6: return protobuf_js_6_common.loadObject(value, options); + case 5: var deprecation_message = 'Calling grpc.loadObject with an object ' + 'generated by ProtoBuf.js 5 is deprecated. Please upgrade to ' + 'ProtoBuf.js 6.'; common.log(grpc.logVerbosity.INFO, deprecation_message); return protobuf_js_5_common.loadObject(value, options); + default: + throw new Error('Unrecognized protobufjsVersion', protobufjsVersion); } }; @@ -90,9 +125,8 @@ function applyProtoRoot(filename, root) { /** * Load a gRPC object from a .proto file. The options object can provide the * following options: - * - convertFieldsToCamelCase: Loads this file with that option on protobuf.js - * set as specified. See - * https://github.com/dcodeIO/protobuf.js/wiki/Advanced-options for details + * - convertFieldsToCamelCase: Load this file with field names in camel case + * instead of their original case * - binaryAsBase64: deserialize bytes values as base64 strings instead of * Buffers. Defaults to false * - longsAsStrings: deserialize long values as strings instead of objects. @@ -113,6 +147,7 @@ exports.load = function load(filename, format, options) { still the possibility of adding other formats that would be loaded differently */ options = _.defaults(options, common.defaultGrpcOptions); + options.protobufjs_version = 6; var root = new ProtoBuf.Root(); var parse_options = {keepCase: !options.convertFieldsToCamelCase}; return loadObject(root.loadSync(applyProtoRoot(filename, root), diff --git a/src/node/src/protobuf_js_5_common.js b/src/node/src/protobuf_js_5_common.js index aecb25339bf..b16228b9ee8 100644 --- a/src/node/src/protobuf_js_5_common.js +++ b/src/node/src/protobuf_js_5_common.js @@ -165,3 +165,16 @@ exports.loadObject = function loadObject(value, options) { return value; } }; + +/** + * The primary purpose of this method is to distinguish between reflection + * objects from different versions of ProtoBuf.js. This is just a heuristic, + * checking for properties that are (currently) specific to this version of + * ProtoBuf.js + * @param {Object} obj The object to check + * @return {boolean} Whether the object appears to be a Protobuf.js 5 + * ReflectionObject + */ +exports.isProbablyProtobufJs5 = function isProbablyProtobufJs5(obj) { + return _.isArray(obj.children) && (typeof obj.build === 'function'); +}; diff --git a/src/node/src/protobuf_js_6_common.js b/src/node/src/protobuf_js_6_common.js index ddee0ea9ee3..5e2b94e1e21 100644 --- a/src/node/src/protobuf_js_6_common.js +++ b/src/node/src/protobuf_js_6_common.js @@ -157,3 +157,16 @@ exports.loadObject = function loadObject(value, options) { // Otherwise, it's not something we need to change return value; }; + +/** + * The primary purpose of this method is to distinguish between reflection + * objects from different versions of ProtoBuf.js. This is just a heuristic, + * checking for properties that are (currently) specific to this version of + * ProtoBuf.js + * @param {Object} obj The object to check + * @return {boolean} Whether the object appears to be a Protobuf.js 6 + * ReflectionObject + */ +exports.isProbablyProtobufJs6 = function isProbablyProtobufJs6(obj) { + return (typeof obj.root === 'object') && (typeof obj.resolve === 'function'); +}; diff --git a/src/node/src/server.js b/src/node/src/server.js index 9a149ebf0a5..77d50b82505 100644 --- a/src/node/src/server.js +++ b/src/node/src/server.js @@ -781,18 +781,16 @@ Server.prototype.addService = function(service, implementation) { */ Server.prototype.addProtoService = function(service, implementation) { var options; + var protobuf_js_5_common = require('./protobuf_js_5_common'); + var protobuf_js_6_common = require('./protobuf_js_6_common'); common.log(grpc.logVerbosity.INFO, 'Server#addProtoService is deprecated. Use addService instead'); - if (service.hasOwnProperty('children')) { - // Heuristically: this is a Protobuf.js 5 service object - var protobuf_js_5_common = require('./protobuf_js_5_common'); + if (protobuf_js_5_common.isProbablyProtobufJs5(service)) { options = _.defaults(service.grpc_options, common.defaultGrpcOptions); this.addService( protobuf_js_5_common.getProtobufServiceAttrs(service, options), implementation); - } else if (service.hasOwnProperty('methods')) { - // Heuristically: this is a Protobuf.js 6 service object - var protobuf_js_6_common = require('./protobuf_js_6_common'); + } else if (protobuf_js_6_common.isProbablyProtobufJs6(service)) { options = _.defaults(service.grpc_options, common.defaultGrpcOptions); this.addService( protobuf_js_6_common.getProtobufServiceAttrs(service, options), diff --git a/src/node/test/common_test.js b/src/node/test/common_test.js index f9efd8231d8..4b7a8c22537 100644 --- a/src/node/test/common_test.js +++ b/src/node/test/common_test.js @@ -155,3 +155,27 @@ describe('Proto message bytes serialize and deserialize', function() { assert.deepEqual(unpackedDeserialized, expectedDeserialize); }); }); +describe('Proto message oneof serialize and deserialize', function() { + var oneofSerialize = serializeCls(messages_proto.OneOfValues); + var oneofDeserialize = deserializeCls( + messages_proto.OneOfValues, default_options); + it('Should have idempotent round trips', function() { + var test_message = {oneof_choice: 'int_choice', int_choice: 5}; + var serialized1 = oneofSerialize(test_message); + var deserialized1 = oneofDeserialize(serialized1); + assert.equal(deserialized1.int_choice, 5); + var serialized2 = oneofSerialize(deserialized1); + var deserialized2 = oneofDeserialize(serialized2); + assert.deepEqual(deserialized1, deserialized2); + }); + it('Should emit a property indicating which field was chosen', function() { + var test_message1 = {oneof_choice: 'int_choice', int_choice: 5}; + var serialized1 = oneofSerialize(test_message1); + var deserialized1 = oneofDeserialize(serialized1); + assert.equal(deserialized1.oneof_choice, 'int_choice'); + var test_message2 = {oneof_choice: 'string_choice', string_choice: 'abc'}; + var serialized2 = oneofSerialize(test_message2); + var deserialized2 = oneofDeserialize(serialized2); + assert.equal(deserialized2.oneof_choice, 'int_choice'); + }); +}); diff --git a/src/node/test/test_messages.proto b/src/node/test/test_messages.proto index a1a6a32833d..2cd133161b7 100644 --- a/src/node/test/test_messages.proto +++ b/src/node/test/test_messages.proto @@ -41,3 +41,10 @@ message SequenceValues { bytes bytes_field = 1; repeated int32 repeated_field = 2; } + +message OneOfValues { + oneof oneof_choice { + int32 int_choice = 1; + string string_choice = 2; + } +} \ No newline at end of file From c1b758f48859dbfe3ff8fe0c3e9b6adf00a711b8 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 21 Mar 2017 18:21:05 -0700 Subject: [PATCH 12/90] Use new oneofs option in deserialize call --- src/node/src/protobuf_js_6_common.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/node/src/protobuf_js_6_common.js b/src/node/src/protobuf_js_6_common.js index 5e2b94e1e21..6a85e4ac23d 100644 --- a/src/node/src/protobuf_js_6_common.js +++ b/src/node/src/protobuf_js_6_common.js @@ -50,7 +50,8 @@ exports.deserializeCls = function deserializeCls(cls, options) { defaults: true, bytes: options.binaryAsBase64 ? String : Buffer, longs: options.longsAsStrings ? String : null, - enums: String + enums: String, + oneofs: true }; /** * Deserialize a buffer to a message object From dbd36187824e7b7d17be0c182187c1ac5952b59f Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 21 Mar 2017 18:59:49 -0700 Subject: [PATCH 13/90] Strategy change --- src/core/lib/slice/slice_buffer.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/core/lib/slice/slice_buffer.c b/src/core/lib/slice/slice_buffer.c index 451319c50d6..c96b9c3b281 100644 --- a/src/core/lib/slice/slice_buffer.c +++ b/src/core/lib/slice/slice_buffer.c @@ -51,19 +51,23 @@ static void maybe_embiggen(grpc_slice_buffer *sb) { size_t slice_count = sb->count + slice_offset; if (slice_count == sb->capacity) { - sb->capacity = GROW(sb->capacity); - GPR_ASSERT(sb->capacity > slice_count); - if (sb->base_slices == sb->inlined) { - sb->base_slices = gpr_malloc(sb->capacity * sizeof(grpc_slice)); - memcpy(sb->base_slices, sb->inlined, slice_count * sizeof(grpc_slice)); - sb->slices = sb->base_slices + slice_offset; + if (sb->base_slices != sb->slices) { + /* Make room by moving elements if there's still space unused */ + memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice)); + sb->slices = sb->base_slices; } else { - if (sb->base_slices != sb->slices) { - memmove(sb->base_slices, sb->slices, sb->count * sizeof(grpc_slice)); + /* Allocate more memory if no more space is available */ + sb->capacity = GROW(sb->capacity); + GPR_ASSERT(sb->capacity > slice_count); + if (sb->base_slices == sb->inlined) { + sb->base_slices = gpr_malloc(sb->capacity * sizeof(grpc_slice)); + memcpy(sb->base_slices, sb->inlined, slice_count * sizeof(grpc_slice)); + } else { + sb->base_slices = + gpr_realloc(sb->base_slices, sb->capacity * sizeof(grpc_slice)); } - sb->base_slices = - gpr_realloc(sb->base_slices, sb->capacity * sizeof(grpc_slice)); - sb->slices = sb->base_slices; + + sb->slices = sb->base_slices + slice_offset; } } } From e76604fc3607cc49cf7256caec665ce3da630671 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 23 Mar 2017 20:27:26 +0100 Subject: [PATCH 14/90] link grpc_csharp_ext with static runtime --- tools/run_tests/helper_scripts/pre_build_csharp.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.bat b/tools/run_tests/helper_scripts/pre_build_csharp.bat index 99df1c66268..bee430ac868 100644 --- a/tools/run_tests/helper_scripts/pre_build_csharp.bat +++ b/tools/run_tests/helper_scripts/pre_build_csharp.bat @@ -43,7 +43,7 @@ cd build mkdir %ARCHITECTURE% cd %ARCHITECTURE% @rem TODO(jtattermusch): Stop hardcoding path to yasm once Jenkins workers can locate yasm correctly -cmake -G "Visual Studio 14 2015" -A %ARCHITECTURE% -DgRPC_BUILD_TESTS=OFF -DCMAKE_ASM_NASM_COMPILER="C:/Program Files (x86)/yasm/yasm.exe" ../../.. || goto :error +cmake -G "Visual Studio 14 2015" -A %ARCHITECTURE% -DgRPC_BUILD_TESTS=OFF -DgRPC_MSVC_STATIC_RUNTIME=ON -DCMAKE_ASM_NASM_COMPILER="C:/Program Files (x86)/yasm/yasm.exe" ../../.. || goto :error cd ..\..\.. @rem Location of nuget.exe From fef391360cfb470355cb6ec00f247ccbd885ef0e Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Thu, 23 Mar 2017 17:45:21 -0700 Subject: [PATCH 15/90] Node: explicitly define Release and Debug builds, use build flags from build.yaml --- binding.gyp | 76 +++++++++++++------- templates/binding.gyp.template | 64 ++++++++++------- tools/run_tests/helper_scripts/build_node.sh | 3 +- 3 files changed, 94 insertions(+), 49 deletions(-) diff --git a/binding.gyp b/binding.gyp index 2ec57768ae7..f941dc75a94 100644 --- a/binding.gyp +++ b/binding.gyp @@ -47,9 +47,42 @@ # Some Node installations use the system installation of OpenSSL, and on # some systems, the system OpenSSL still does not have ALPN support. This # will let users recompile gRPC to work without ALPN. - 'grpc_alpn%': 'true' + 'grpc_alpn%': 'true', + # Indicates that the library should be built with gcov. + 'grpc_gcov%': 'false' }, 'target_defaults': { + 'configurations': { + 'Release': { + 'cflags': [ + '-O2', + ], + 'defines': [ + 'NDEBUG', + ], + }, + 'Debug': { + 'cflags': [ + '-O0', + ], + 'defines': [ + '_DEBUG', + 'DEBUG', + ], + }, + }, + 'cflags': [ + '-g', + '-Wall', + '-Wextra', + '-Werror', + '-Wno-long-long', + '-Wno-unused-parameter', + '-DOSATOMIC_USE_INLINED=1', + ], + 'ldflags': [ + '-g', + ], 'include_dirs': [ '.', 'include' @@ -63,6 +96,24 @@ 'GRPC_UV' ] }], + ['grpc_gcov=="true"', { + 'cflags': [ + '-O0', + '-fprofile-arcs', + '-ftest-coverage', + '-Wno-return-type', + ], + 'defines': [ + '_DEBUG', + 'DEBUG', + 'GPR_GCOV', + ], + 'ldflags': [ + '-fprofile-arcs', + '-ftest-coverage', + '-rdynamic', + ], + }], ['OS!="win" and runtime=="electron"', { "defines": [ 'OPENSSL_NO_THREADS' @@ -124,25 +175,8 @@ "ws2_32" ] }, { # OS != "win" - 'variables': { - 'config': ' Date: Thu, 23 Mar 2017 18:00:09 -0700 Subject: [PATCH 16/90] Add enumsAsStrings option, as the original upgrade PR did --- src/node/index.js | 4 ++++ src/node/src/common.js | 1 + src/node/src/protobuf_js_6_common.js | 2 +- src/node/test/common_test.js | 22 ++++++++++++++++++++++ src/node/test/test_messages.proto | 10 ++++++++++ 5 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/node/index.js b/src/node/index.js index ba73e3dac8f..071bfd7927b 100644 --- a/src/node/index.js +++ b/src/node/index.js @@ -64,6 +64,8 @@ grpc.setDefaultRootsPem(fs.readFileSync(SSL_ROOTS_PATH, 'ascii')); * Buffers. Defaults to false * - longsAsStrings: deserialize long values as strings instead of objects. * Defaults to true + * - enumsAsStrings: deserialize enum values as strings instead of numbers. + * Defaults to true * - deprecatedArgumentOrder: Use the beta method argument order for client * methods, with optional arguments after the callback. Defaults to false. * This option is only a temporary stopgap measure to smooth an API breakage. @@ -131,6 +133,8 @@ function applyProtoRoot(filename, root) { * Buffers. Defaults to false * - longsAsStrings: deserialize long values as strings instead of objects. * Defaults to true + * - enumsAsStrings: deserialize enum values as strings instead of numbers. + * Defaults to true * - deprecatedArgumentOrder: Use the beta method argument order for client * methods, with optional arguments after the callback. Defaults to false. * This option is only a temporary stopgap measure to smooth an API breakage. diff --git a/src/node/src/common.js b/src/node/src/common.js index 93df5138172..757969dbddb 100644 --- a/src/node/src/common.js +++ b/src/node/src/common.js @@ -87,5 +87,6 @@ exports.defaultGrpcOptions = { convertFieldsToCamelCase: false, binaryAsBase64: false, longsAsStrings: true, + enumsAsStrings: true, deprecatedArgumentOrder: false }; diff --git a/src/node/src/protobuf_js_6_common.js b/src/node/src/protobuf_js_6_common.js index 6a85e4ac23d..baa62cce866 100644 --- a/src/node/src/protobuf_js_6_common.js +++ b/src/node/src/protobuf_js_6_common.js @@ -50,7 +50,7 @@ exports.deserializeCls = function deserializeCls(cls, options) { defaults: true, bytes: options.binaryAsBase64 ? String : Buffer, longs: options.longsAsStrings ? String : null, - enums: String, + enums: options.enumsAsStrings ? String : null, oneofs: true }; /** diff --git a/src/node/test/common_test.js b/src/node/test/common_test.js index 4b7a8c22537..39ff6a5f1fc 100644 --- a/src/node/test/common_test.js +++ b/src/node/test/common_test.js @@ -179,3 +179,25 @@ describe('Proto message oneof serialize and deserialize', function() { assert.equal(deserialized2.oneof_choice, 'int_choice'); }); }); +describe('Proto message enum serialize and deserialize', function() { + var enumSerialize = serializeCls(messages_proto.EnumValues); + var enumDeserialize = deserializeCls( + messages_proto.EnumValues, default_options); + var enumIntOptions = _.defaults({enumsAsStrings: false}, default_options); + var enumIntDeserialize = deserializeCls( + messages_proto.EnumValues, enumIntOptions); + it('Should accept both names and numbers', function() { + var nameSerialized = enumSerialize({enum_value: 'ONE'}); + var numberSerialized = enumSerialize({enum_value: 1}); + assert.strictEqual(messages_proto.TestEnum.ONE, 1); + assert.deepEqual(enumDeserialize(nameSerialized), + enumDeserialize(numberSerialized)); + }); + it('Should deserialize as a string the enumsAsStrings option', function() { + var serialized = enumSerialize({enum_value: 'TWO'}); + var nameDeserialized = enumDeserialize(serialized); + var numberDeserialized = enumIntDeserialize(serialized); + assert.deepEqual(nameDeserialized, {enum_value: 'TWO'}); + assert.deepEqual(numberDeserialized, {enum_value: 2}); + }); +}); diff --git a/src/node/test/test_messages.proto b/src/node/test/test_messages.proto index 2cd133161b7..ae70f6e152a 100644 --- a/src/node/test/test_messages.proto +++ b/src/node/test/test_messages.proto @@ -47,4 +47,14 @@ message OneOfValues { int32 int_choice = 1; string string_choice = 2; } +} + +enum TestEnum { + ZERO = 0; + ONE = 1; + TWO = 2; +} + +message EnumValues { + TestEnum enum_value = 1; } \ No newline at end of file From 27338de44531467941f5dafb823560c0e1e3d401 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 23 Mar 2017 18:23:49 -0700 Subject: [PATCH 17/90] add test in that sends a sigint to client while its making an rpc ona child thread - segfaults on mac --- .../end2end/killed_client_thread_client.rb | 55 ++++++++ .../end2end/killed_client_thread_driver.rb | 132 ++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100755 src/ruby/end2end/killed_client_thread_client.rb create mode 100755 src/ruby/end2end/killed_client_thread_driver.rb diff --git a/src/ruby/end2end/killed_client_thread_client.rb b/src/ruby/end2end/killed_client_thread_client.rb new file mode 100755 index 00000000000..9a69161736e --- /dev/null +++ b/src/ruby/end2end/killed_client_thread_client.rb @@ -0,0 +1,55 @@ +#!/usr/bin/env ruby + +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +require_relative './end2end_common' + +def main + server_port = '' + OptionParser.new do |opts| + opts.on('--client_control_port=P', String) do + STDERR.puts 'client control port not used' + end + opts.on('--server_port=P', String) do |p| + server_port = p + end + end.parse! + + thd = Thread.new do + stub = Echo::EchoServer::Stub.new("localhost:#{server_port}", + :this_channel_is_insecure) + stub.echo(Echo::EchoRequest.new(request: 'hello')) + raise 'the clients rpc in this test shouldnt complete. ' \ + 'expecting SIGINT to happen in the middle of the call' + end + thd.join +end + +main diff --git a/src/ruby/end2end/killed_client_thread_driver.rb b/src/ruby/end2end/killed_client_thread_driver.rb new file mode 100755 index 00000000000..71d5db276df --- /dev/null +++ b/src/ruby/end2end/killed_client_thread_driver.rb @@ -0,0 +1,132 @@ +#!/usr/bin/env ruby + +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +require_relative './end2end_common' + +class SleepingEchoServerImpl < Echo::EchoServer::Service + def initialize(call_started, call_started_mu, call_started_cv) + @call_started = call_started + @call_started_mu = call_started_mu + @call_started_cv = call_started_cv + end + def echo(echo_req, _) + @call_started_mu.synchronize do + @call_started.set_true + @call_started_cv.signal + end + sleep 1000 + Echo::EchoReply.new(response: echo_req.request) + end +end + +class SleepingServerRunner + def initialize(service_impl) + @service_impl = service_impl + end + + def run + @srv = GRPC::RpcServer.new + port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure) + @srv.handle(@service_impl) + + @thd = Thread.new do + @srv.run + end + @srv.wait_till_running + port + end + + def stop + @srv.stop + @thd.join + fail 'server not stopped' unless @srv.stopped? + end +end + +class BoolHolder + def init + @val = false + end + def set_true + @val = true + end + def get_val + @val + end +end + + +def main + STDERR.puts 'start server' + + call_started = BoolHolder.new + call_started_mu = Mutex.new + call_started_cv = ConditionVariable.new + + service_impl = SleepingEchoServerImpl.new(call_started, call_started_mu, call_started_cv) + server_runner = SleepingServerRunner.new(service_impl) + server_port = server_runner.run + + STDERR.puts 'start client' + _, client_pid = start_client('killed_client_thread_client.rb', + server_port) + + call_started_mu.synchronize do + while !call_started.get_val do + call_started_cv.wait(call_started_mu) + end + end + + Process.kill('SIGINT', client_pid) + STDERR.puts 'sent shutdown' + + begin + Timeout.timeout(10) do + Process.wait(client_pid) + end + rescue Timeout::Error + STDERR.puts "timeout wait for client pid #{client_pid}" + Process.kill('SIGKILL', client_pid) + Process.wait(client_pid) + STDERR.puts 'killed client child' + raise 'Timed out waiting for client process. It likely hangs when ' \ + 'killed while in the middle of an rpc' + end + + client_exit_code = $CHILD_STATUS + if client_exit_code.termsig != 2 # SIGINT + fail "expected client exit from SIGINT but got child status: #{client_exit_code}" + end + + server_runner.stop +end + +main From 34bb6df108742cf5c17d019ee5d010c7ff12a55b Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Thu, 23 Mar 2017 22:18:02 -0700 Subject: [PATCH 18/90] allocated run batch stack on the heap --- src/ruby/ext/grpc/rb_call.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c index 82d340b2544..cf62564667a 100644 --- a/src/ruby/ext/grpc/rb_call.c +++ b/src/ruby/ext/grpc/rb_call.c @@ -784,7 +784,7 @@ static VALUE grpc_run_batch_stack_build_result(run_batch_stack *st) { Only one operation of each type can be active at once in any given batch */ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE ops_hash) { - run_batch_stack st; + run_batch_stack *st; grpc_rb_call *call = NULL; grpc_event ev; grpc_call_error err; @@ -792,6 +792,8 @@ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE ops_hash) { VALUE rb_write_flag = rb_ivar_get(self, id_write_flag); unsigned write_flag = 0; void *tag = (void*)&st; + st = gpr_malloc(sizeof(run_batch_stack)); + if (RTYPEDDATA_DATA(self) == NULL) { rb_raise(grpc_rb_eCallError, "Cannot run batch on closed call"); return Qnil; @@ -806,14 +808,15 @@ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE ops_hash) { if (rb_write_flag != Qnil) { write_flag = NUM2UINT(rb_write_flag); } - grpc_run_batch_stack_init(&st, write_flag); - grpc_run_batch_stack_fill_ops(&st, ops_hash); + grpc_run_batch_stack_init(st, write_flag); + grpc_run_batch_stack_fill_ops(st, ops_hash); /* call grpc_call_start_batch, then wait for it to complete using * pluck_event */ - err = grpc_call_start_batch(call->wrapped, st.ops, st.op_num, tag, NULL); + err = grpc_call_start_batch(call->wrapped, st->ops, st->op_num, tag, NULL); if (err != GRPC_CALL_OK) { - grpc_run_batch_stack_cleanup(&st); + grpc_run_batch_stack_cleanup(st); + gpr_free(st); rb_raise(grpc_rb_eCallError, "grpc_call_start_batch failed with %s (code=%d)", grpc_call_error_detail_of(err), err); @@ -826,8 +829,9 @@ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE ops_hash) { } /* Build and return the BatchResult struct result, if there is an error, it's reflected in the status */ - result = grpc_run_batch_stack_build_result(&st); - grpc_run_batch_stack_cleanup(&st); + result = grpc_run_batch_stack_build_result(st); + grpc_run_batch_stack_cleanup(st); + gpr_free(st); return result; } From 077f890965766b13d3a5d19a1589b987e47c0570 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Fri, 24 Mar 2017 09:53:40 -0700 Subject: [PATCH 19/90] conform test to formatter --- src/ruby/end2end/channel_closing_driver.rb | 2 +- src/ruby/end2end/channel_state_driver.rb | 2 +- src/ruby/end2end/end2end_common.rb | 5 +- .../end2end/killed_client_thread_client.rb | 5 +- .../end2end/killed_client_thread_driver.rb | 56 +++++++------------ src/ruby/end2end/sig_handling_driver.rb | 2 +- .../sig_int_during_channel_watch_driver.rb | 2 +- .../helper_scripts/run_ruby_end2end_tests.sh | 1 + 8 files changed, 31 insertions(+), 44 deletions(-) diff --git a/src/ruby/end2end/channel_closing_driver.rb b/src/ruby/end2end/channel_closing_driver.rb index 43e2fe8cbbd..d3e5373b0bb 100755 --- a/src/ruby/end2end/channel_closing_driver.rb +++ b/src/ruby/end2end/channel_closing_driver.rb @@ -36,7 +36,7 @@ require_relative './end2end_common' def main STDERR.puts 'start server' - server_runner = ServerRunner.new + server_runner = ServerRunner.new(EchoServerImpl) server_port = server_runner.run sleep 1 diff --git a/src/ruby/end2end/channel_state_driver.rb b/src/ruby/end2end/channel_state_driver.rb index c3184bf9392..80fb62899e5 100755 --- a/src/ruby/end2end/channel_state_driver.rb +++ b/src/ruby/end2end/channel_state_driver.rb @@ -35,7 +35,7 @@ require_relative './end2end_common' def main STDERR.puts 'start server' - server_runner = ServerRunner.new + server_runner = ServerRunner.new(EchoServerImpl) server_port = server_runner.run sleep 1 diff --git a/src/ruby/end2end/end2end_common.rb b/src/ruby/end2end/end2end_common.rb index 9534bb20787..1c87ceddf1d 100755 --- a/src/ruby/end2end/end2end_common.rb +++ b/src/ruby/end2end/end2end_common.rb @@ -55,13 +55,14 @@ end # ServerRunner starts an "echo server" that test clients can make calls to class ServerRunner - def initialize + def initialize(service_impl) + @service_impl = service_impl end def run @srv = GRPC::RpcServer.new port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure) - @srv.handle(EchoServerImpl) + @srv.handle(@service_impl) @thd = Thread.new do @srv.run diff --git a/src/ruby/end2end/killed_client_thread_client.rb b/src/ruby/end2end/killed_client_thread_client.rb index 9a69161736e..d5a7db7d586 100755 --- a/src/ruby/end2end/killed_client_thread_client.rb +++ b/src/ruby/end2end/killed_client_thread_client.rb @@ -29,6 +29,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Attempt to reproduce +# https://github.com/GoogleCloudPlatform/google-cloud-ruby/issues/1327 + require_relative './end2end_common' def main @@ -46,7 +49,7 @@ def main stub = Echo::EchoServer::Stub.new("localhost:#{server_port}", :this_channel_is_insecure) stub.echo(Echo::EchoRequest.new(request: 'hello')) - raise 'the clients rpc in this test shouldnt complete. ' \ + fail 'the clients rpc in this test shouldnt complete. ' \ 'expecting SIGINT to happen in the middle of the call' end thd.join diff --git a/src/ruby/end2end/killed_client_thread_driver.rb b/src/ruby/end2end/killed_client_thread_driver.rb index 71d5db276df..2963a287c59 100755 --- a/src/ruby/end2end/killed_client_thread_driver.rb +++ b/src/ruby/end2end/killed_client_thread_driver.rb @@ -31,12 +31,15 @@ require_relative './end2end_common' +# Service that sleeps for a long time upon receiving an 'echo request' +# Also, this notified @call_started_cv once it has received a request. class SleepingEchoServerImpl < Echo::EchoServer::Service def initialize(call_started, call_started_mu, call_started_cv) @call_started = call_started @call_started_mu = call_started_mu @call_started_cv = call_started_cv end + def echo(echo_req, _) @call_started_mu.synchronize do @call_started.set_true @@ -47,43 +50,19 @@ class SleepingEchoServerImpl < Echo::EchoServer::Service end end -class SleepingServerRunner - def initialize(service_impl) - @service_impl = service_impl - end - - def run - @srv = GRPC::RpcServer.new - port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure) - @srv.handle(@service_impl) - - @thd = Thread.new do - @srv.run - end - @srv.wait_till_running - port - end - - def stop - @srv.stop - @thd.join - fail 'server not stopped' unless @srv.stopped? - end -end - +# Mutable boolean class BoolHolder + attr_reader :val + def init @val = false end + def set_true @val = true end - def get_val - @val - end end - def main STDERR.puts 'start server' @@ -91,20 +70,22 @@ def main call_started_mu = Mutex.new call_started_cv = ConditionVariable.new - service_impl = SleepingEchoServerImpl.new(call_started, call_started_mu, call_started_cv) - server_runner = SleepingServerRunner.new(service_impl) + service_impl = SleepingEchoServerImpl.new(call_started, + call_started_mu, + call_started_cv) + server_runner = ServerRunner.new(service_impl) server_port = server_runner.run STDERR.puts 'start client' _, client_pid = start_client('killed_client_thread_client.rb', - server_port) + server_port) call_started_mu.synchronize do - while !call_started.get_val do - call_started_cv.wait(call_started_mu) - end + call_started_cv.wait(call_started_mu) until call_started.val end + # SIGINT the child process not that it's + # in the middle of an RPC (happening on a non-main thread) Process.kill('SIGINT', client_pid) STDERR.puts 'sent shutdown' @@ -117,13 +98,14 @@ def main Process.kill('SIGKILL', client_pid) Process.wait(client_pid) STDERR.puts 'killed client child' - raise 'Timed out waiting for client process. It likely hangs when ' \ - 'killed while in the middle of an rpc' + raise 'Timed out waiting for client process. ' \ + 'It likely hangs when killed while in the middle of an rpc' end client_exit_code = $CHILD_STATUS if client_exit_code.termsig != 2 # SIGINT - fail "expected client exit from SIGINT but got child status: #{client_exit_code}" + fail 'expected client exit from SIGINT ' \ + "but got child status: #{client_exit_code}" end server_runner.stop diff --git a/src/ruby/end2end/sig_handling_driver.rb b/src/ruby/end2end/sig_handling_driver.rb index c5d46e074cb..6691464dc69 100755 --- a/src/ruby/end2end/sig_handling_driver.rb +++ b/src/ruby/end2end/sig_handling_driver.rb @@ -36,7 +36,7 @@ require_relative './end2end_common' def main STDERR.puts 'start server' - server_runner = ServerRunner.new + server_runner = ServerRunner.new(EchoServerImpl) server_port = server_runner.run sleep 1 diff --git a/src/ruby/end2end/sig_int_during_channel_watch_driver.rb b/src/ruby/end2end/sig_int_during_channel_watch_driver.rb index 84d039bf19d..670cda0919f 100755 --- a/src/ruby/end2end/sig_int_during_channel_watch_driver.rb +++ b/src/ruby/end2end/sig_int_during_channel_watch_driver.rb @@ -36,7 +36,7 @@ require_relative './end2end_common' def main STDERR.puts 'start server' - server_runner = ServerRunner.new + server_runner = ServerRunner.new(EchoServerImpl) server_port = server_runner.run sleep 1 diff --git a/tools/run_tests/helper_scripts/run_ruby_end2end_tests.sh b/tools/run_tests/helper_scripts/run_ruby_end2end_tests.sh index d7da6364d8f..92d69757073 100755 --- a/tools/run_tests/helper_scripts/run_ruby_end2end_tests.sh +++ b/tools/run_tests/helper_scripts/run_ruby_end2end_tests.sh @@ -38,4 +38,5 @@ ruby src/ruby/end2end/sig_handling_driver.rb || EXIT_CODE=1 ruby src/ruby/end2end/channel_state_driver.rb || EXIT_CODE=1 ruby src/ruby/end2end/channel_closing_driver.rb || EXIT_CODE=1 ruby src/ruby/end2end/sig_int_during_channel_watch_driver.rb || EXIT_CODE=1 +ruby src/ruby/end2end/killed_client_thread_driver.rb || EXIT_CODE=1 exit $EXIT_CODE From 8d8dce8db31dc9f54f606417ed6bc1ec93f9d76b Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Fri, 24 Mar 2017 10:32:15 -0700 Subject: [PATCH 20/90] malloc run_batch_stack after type checks --- src/ruby/ext/grpc/rb_call.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c index cf62564667a..344cb941ffb 100644 --- a/src/ruby/ext/grpc/rb_call.c +++ b/src/ruby/ext/grpc/rb_call.c @@ -784,7 +784,7 @@ static VALUE grpc_run_batch_stack_build_result(run_batch_stack *st) { Only one operation of each type can be active at once in any given batch */ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE ops_hash) { - run_batch_stack *st; + run_batch_stack *st = NULL; grpc_rb_call *call = NULL; grpc_event ev; grpc_call_error err; @@ -792,7 +792,6 @@ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE ops_hash) { VALUE rb_write_flag = rb_ivar_get(self, id_write_flag); unsigned write_flag = 0; void *tag = (void*)&st; - st = gpr_malloc(sizeof(run_batch_stack)); if (RTYPEDDATA_DATA(self) == NULL) { rb_raise(grpc_rb_eCallError, "Cannot run batch on closed call"); @@ -808,6 +807,7 @@ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE ops_hash) { if (rb_write_flag != Qnil) { write_flag = NUM2UINT(rb_write_flag); } + st = gpr_malloc(sizeof(run_batch_stack)); grpc_run_batch_stack_init(st, write_flag); grpc_run_batch_stack_fill_ops(st, ops_hash); From 4364ded9b16c5a0e1259af7423780c254d0465e6 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Fri, 24 Mar 2017 10:41:37 -0700 Subject: [PATCH 21/90] wording fix in comments --- src/ruby/end2end/killed_client_thread_driver.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ruby/end2end/killed_client_thread_driver.rb b/src/ruby/end2end/killed_client_thread_driver.rb index 2963a287c59..f76d3e1746d 100755 --- a/src/ruby/end2end/killed_client_thread_driver.rb +++ b/src/ruby/end2end/killed_client_thread_driver.rb @@ -32,7 +32,7 @@ require_relative './end2end_common' # Service that sleeps for a long time upon receiving an 'echo request' -# Also, this notified @call_started_cv once it has received a request. +# Also, this notifies @call_started_cv once it has received a request. class SleepingEchoServerImpl < Echo::EchoServer::Service def initialize(call_started, call_started_mu, call_started_cv) @call_started = call_started @@ -84,7 +84,7 @@ def main call_started_cv.wait(call_started_mu) until call_started.val end - # SIGINT the child process not that it's + # SIGINT the child process now that it's # in the middle of an RPC (happening on a non-main thread) Process.kill('SIGINT', client_pid) STDERR.puts 'sent shutdown' From 69c2717c808b297ff1024e31ae45eb5b588cffa7 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Fri, 24 Mar 2017 11:39:52 -0700 Subject: [PATCH 22/90] bump v1.2.x branch version to 1.2.1-pre2 --- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Auth/project.json | 4 ++-- src/csharp/Grpc.Core.Testing/project.json | 4 ++-- src/csharp/Grpc.Core/VersionInfo.cs | 2 +- src/csharp/Grpc.Core/project.json | 2 +- src/csharp/Grpc.HealthCheck/project.json | 4 ++-- src/csharp/Grpc.Reflection/project.json | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 31 files changed, 39 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a0787e2f1a..1543140c0aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.2.1-pre1") +set(PACKAGE_VERSION "1.2.1-pre2") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 4a32fed2c96..041e4f5c7c5 100644 --- a/Makefile +++ b/Makefile @@ -412,8 +412,8 @@ Q = @ endif CORE_VERSION = 3.0.0-dev -CPP_VERSION = 1.2.1-pre1 -CSHARP_VERSION = 1.2.1-pre1 +CPP_VERSION = 1.2.1-pre2 +CSHARP_VERSION = 1.2.1-pre2 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 1ec134645ce..49d64bd1da0 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 3.0.0-dev g_stands_for: green - version: 1.2.1-pre1 + version: 1.2.1-pre2 filegroups: - name: census public_headers: diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index c502c4440eb..fe1f3e008b8 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -37,7 +37,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.2.1-pre1' + version = '1.2.1-pre2' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 5fddf8fd820..b5c172ee82e 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.2.1-pre1' + version = '1.2.1-pre2' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 84478f56ea7..6c2736cd78e 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.2.1-pre1' + version = '1.2.1-pre2' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'http://www.grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index dc1dd38a6e8..7031b3d1449 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -35,7 +35,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.2.1-pre1' + version = '1.2.1-pre2' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' diff --git a/package.json b/package.json index 004f968ea2f..e75dfa88206 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.2.1-pre1", + "version": "1.2.1-pre2", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "http://www.grpc.io/", diff --git a/package.xml b/package.xml index fc3d0923728..44a15c23a96 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2017-03-01 - 1.2.1RC1 - 1.2.1RC1 + 1.2.1RC2 + 1.2.1RC2 beta diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index a40f8f0a284..593e17e7706 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -37,5 +37,5 @@ #include namespace grpc { -grpc::string Version() { return "1.2.1-pre1"; } +grpc::string Version() { return "1.2.1-pre2"; } } diff --git a/src/csharp/Grpc.Auth/project.json b/src/csharp/Grpc.Auth/project.json index 16584f9b14b..6345946c39c 100644 --- a/src/csharp/Grpc.Auth/project.json +++ b/src/csharp/Grpc.Auth/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre1", + "version": "1.2.1-pre2", "title": "gRPC C# Auth", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1-pre1", + "Grpc.Core": "1.2.1-pre2", "Google.Apis.Auth": "1.21.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Core.Testing/project.json b/src/csharp/Grpc.Core.Testing/project.json index 960774c0d95..7337e2d15a8 100644 --- a/src/csharp/Grpc.Core.Testing/project.json +++ b/src/csharp/Grpc.Core.Testing/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre1", + "version": "1.2.1-pre2", "title": "gRPC C# Core Testing", "authors": [ "Google Inc." ], "copyright": "Copyright 2017, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1-pre1" + "Grpc.Core": "1.2.1-pre2" }, "frameworks": { "net45": { diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 25be59a61ce..976508dfac2 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -53,6 +53,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.2.1-pre1"; + public const string CurrentVersion = "1.2.1-pre2"; } } diff --git a/src/csharp/Grpc.Core/project.json b/src/csharp/Grpc.Core/project.json index 5338011dd25..d0eb7759782 100644 --- a/src/csharp/Grpc.Core/project.json +++ b/src/csharp/Grpc.Core/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre1", + "version": "1.2.1-pre2", "title": "gRPC C# Core", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", diff --git a/src/csharp/Grpc.HealthCheck/project.json b/src/csharp/Grpc.HealthCheck/project.json index 2a7ec78852d..ff5429dba6b 100644 --- a/src/csharp/Grpc.HealthCheck/project.json +++ b/src/csharp/Grpc.HealthCheck/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre1", + "version": "1.2.1-pre2", "title": "gRPC C# Healthchecking", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1-pre1", + "Grpc.Core": "1.2.1-pre2", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Reflection/project.json b/src/csharp/Grpc.Reflection/project.json index 18bc2d8d699..ac7696e9074 100644 --- a/src/csharp/Grpc.Reflection/project.json +++ b/src/csharp/Grpc.Reflection/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre1", + "version": "1.2.1-pre2", "title": "gRPC C# Reflection", "authors": [ "Google Inc." ], "copyright": "Copyright 2016, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1-pre1", + "Grpc.Core": "1.2.1-pre2", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index e4f75d3ba01..8a6f22f1fa3 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -28,7 +28,7 @@ @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @rem Current package versions -set VERSION=1.2.1-pre1 +set VERSION=1.2.1-pre2 set PROTOBUF_VERSION=3.0.0 @rem Adjust the location of nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index a5a6b6108a8..56e24de1993 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -66,7 +66,7 @@ dotnet pack --configuration Release Grpc.Auth/project.json --output ../../artifa dotnet pack --configuration Release Grpc.HealthCheck/project.json --output ../../artifacts dotnet pack --configuration Release Grpc.Reflection/project.json --output ../../artifacts -nuget pack Grpc.nuspec -Version "1.2.1-pre1" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.2.1-pre1" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.2.1-pre2" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.2.1-pre2" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index a3966a4abbf..630b3cba006 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.2.1-pre1", + "version": "1.2.1-pre2", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.2.1-pre1", + "grpc": "^1.2.1-pre2", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index de64cca844b..98863c1d776 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.2.1-pre1", + "version": "1.2.1-pre2", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "http://www.grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 906fea86b71..636b9f792aa 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.2.1-pre1' + v = '1.2.1-pre2' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index a623d2d772c..9339adce431 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -38,4 +38,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.2.1-pre1" +#define GRPC_OBJC_VERSION_STRING @"1.2.1-pre2" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 5a0cf4df2e3..418ca993bc7 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.2.1rc1' +VERSION='1.2.1rc2' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index c401cdbbcaa..00b7680a0f5 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.2.1rc1' +VERSION='1.2.1rc2' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 83f5d950b14..cf42d123ae3 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.2.1rc1' +VERSION='1.2.1rc2' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 3613aeee130..acd2b85c95e 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.2.1rc1' +VERSION='1.2.1rc2' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index e9759ad206e..7d54034a298 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -29,5 +29,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.2.1.pre1' + VERSION = '1.2.1.pre2' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index a11e53cb2be..d6a7be76efa 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -29,6 +29,6 @@ module GRPC module Tools - VERSION = '1.2.1.pre1' + VERSION = '1.2.1.pre2' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 73f06ba875d..fede4440800 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.2.1rc1' +VERSION='1.2.1rc2' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 8b69893696b..13734e8d76a 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.1-pre1 +PROJECT_NUMBER = 1.2.1-pre2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 823f9b01688..dfef610a00b 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.1-pre1 +PROJECT_NUMBER = 1.2.1-pre2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 0cb20efaec8169e606f380770daad6257146b661 Mon Sep 17 00:00:00 2001 From: Alexander Polcyn Date: Fri, 24 Mar 2017 15:50:06 -0700 Subject: [PATCH 23/90] update to 1.2.1-pre2 in BUILD file too --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD b/BUILD index db08f57fbf1..faaef5198ab 100644 --- a/BUILD +++ b/BUILD @@ -41,7 +41,7 @@ g_stands_for = "green" core_version = "3.0.0-dev" -version = "1.2.0" +version = "1.2.1-pre2" grpc_cc_library( name = "gpr", From 547cb4b283273a68c61da28f97b149b8c55c8daf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 09:50:33 -0700 Subject: [PATCH 24/90] Make chunk sizes configurable, push channel args down a little deeper --- .../server/insecure/server_chttp2_posix.c | 7 +- src/core/lib/iomgr/endpoint_pair.h | 5 +- src/core/lib/iomgr/endpoint_pair_posix.c | 17 +++-- src/core/lib/iomgr/tcp_client.h | 4 -- src/core/lib/iomgr/tcp_client_posix.c | 24 +------ src/core/lib/iomgr/tcp_posix.c | 70 ++++++++++++++++--- src/core/lib/iomgr/tcp_posix.h | 13 +++- src/core/lib/iomgr/tcp_server_posix.c | 22 ++---- src/core/lib/iomgr/tcp_server_utils_posix.h | 3 +- test/core/bad_client/bad_client.c | 5 +- .../core/end2end/fixtures/h2_sockpair+trace.c | 4 +- test/core/end2end/fixtures/h2_sockpair.c | 4 +- .../core/end2end/fixtures/h2_sockpair_1byte.c | 15 +++- test/core/iomgr/endpoint_pair_test.c | 11 +-- test/core/iomgr/fd_conservation_posix_test.c | 2 +- test/core/iomgr/tcp_posix_test.c | 55 +++++++++------ test/core/security/secure_endpoint_test.c | 11 +-- 17 files changed, 155 insertions(+), 117 deletions(-) diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c index f46e8499326..6ab176e8ad7 100644 --- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c +++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c @@ -57,12 +57,9 @@ void grpc_server_add_insecure_channel_from_fd(grpc_server *server, char *name; gpr_asprintf(&name, "fd:%d", fd); - grpc_resource_quota *resource_quota = grpc_resource_quota_from_channel_args( - grpc_server_get_channel_args(server)); grpc_endpoint *server_endpoint = - grpc_tcp_create(grpc_fd_create(fd, name), resource_quota, - GRPC_TCP_DEFAULT_READ_SLICE_SIZE, name); - grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); + grpc_tcp_create(&exec_ctx, grpc_fd_create(fd, name), + grpc_server_get_channel_args(server), name); gpr_free(name); diff --git a/src/core/lib/iomgr/endpoint_pair.h b/src/core/lib/iomgr/endpoint_pair.h index f9de0c715ec..6407a6ad3f3 100644 --- a/src/core/lib/iomgr/endpoint_pair.h +++ b/src/core/lib/iomgr/endpoint_pair.h @@ -41,8 +41,7 @@ typedef struct { grpc_endpoint *server; } grpc_endpoint_pair; -grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( - const char *name, grpc_resource_quota *resource_quota, - size_t read_slice_size); +grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, + grpc_channel_args *args); #endif /* GRPC_CORE_LIB_IOMGR_ENDPOINT_PAIR_H */ diff --git a/src/core/lib/iomgr/endpoint_pair_posix.c b/src/core/lib/iomgr/endpoint_pair_posix.c index b9ff969e810..5542a372d8b 100644 --- a/src/core/lib/iomgr/endpoint_pair_posix.c +++ b/src/core/lib/iomgr/endpoint_pair_posix.c @@ -62,22 +62,25 @@ static void create_sockets(int sv[2]) { GPR_ASSERT(grpc_set_socket_no_sigpipe_if_possible(sv[1]) == GRPC_ERROR_NONE); } -grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( - const char *name, grpc_resource_quota *resource_quota, - size_t read_slice_size) { +grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, + grpc_channel_args *args) { int sv[2]; grpc_endpoint_pair p; char *final_name; create_sockets(sv); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_asprintf(&final_name, "%s:client", name); - p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name), resource_quota, - read_slice_size, "socketpair-server"); + p.client = grpc_tcp_create(&exec_ctx, grpc_fd_create(sv[1], final_name), args, + "socketpair-server"); gpr_free(final_name); gpr_asprintf(&final_name, "%s:server", name); - p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name), resource_quota, - read_slice_size, "socketpair-client"); + p.server = grpc_tcp_create(&exec_ctx, grpc_fd_create(sv[0], final_name), args, + "socketpair-client"); gpr_free(final_name); + + grpc_exec_ctx_finish(&exec_ctx); return p; } diff --git a/src/core/lib/iomgr/tcp_client.h b/src/core/lib/iomgr/tcp_client.h index 0485661316a..bc367bdfa5d 100644 --- a/src/core/lib/iomgr/tcp_client.h +++ b/src/core/lib/iomgr/tcp_client.h @@ -40,10 +40,6 @@ #include "src/core/lib/iomgr/pollset_set.h" #include "src/core/lib/iomgr/resolve_address.h" -/* Channel arg (integer) setting how large a slice to try and read from the wire - each time recvmsg (or equivalent) is called */ -#define GRPC_ARG_TCP_READ_CHUNK_SIZE "grpc.experimental.tcp_read_chunk_size" - /* Asynchronously connect to an address (specified as (addr, len)), and call cb with arg and the completed connection when done (or call cb with arg and NULL on failure). diff --git a/src/core/lib/iomgr/tcp_client_posix.c b/src/core/lib/iomgr/tcp_client_posix.c index a108b10da6f..a2692707d9e 100644 --- a/src/core/lib/iomgr/tcp_client_posix.c +++ b/src/core/lib/iomgr/tcp_client_posix.c @@ -137,29 +137,7 @@ static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { grpc_endpoint *grpc_tcp_client_create_from_fd( grpc_exec_ctx *exec_ctx, grpc_fd *fd, const grpc_channel_args *channel_args, const char *addr_str) { - size_t tcp_read_chunk_size = GRPC_TCP_DEFAULT_READ_SLICE_SIZE; - grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); - if (channel_args != NULL) { - for (size_t i = 0; i < channel_args->num_args; i++) { - if (0 == - strcmp(channel_args->args[i].key, GRPC_ARG_TCP_READ_CHUNK_SIZE)) { - grpc_integer_options options = {(int)tcp_read_chunk_size, 1, - 8 * 1024 * 1024}; - tcp_read_chunk_size = (size_t)grpc_channel_arg_get_integer( - &channel_args->args[i], options); - } else if (0 == - strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { - grpc_resource_quota_unref_internal(exec_ctx, resource_quota); - resource_quota = grpc_resource_quota_ref_internal( - channel_args->args[i].value.pointer.p); - } - } - } - - grpc_endpoint *ep = - grpc_tcp_create(fd, resource_quota, tcp_read_chunk_size, addr_str); - grpc_resource_quota_unref_internal(exec_ctx, resource_quota); - return ep; + return grpc_tcp_create(exec_ctx, fd, channel_args, addr_str); } static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index 74429e3a566..bfc7de59f20 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -54,6 +54,7 @@ #include #include +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/profiling/timers.h" @@ -86,6 +87,9 @@ typedef struct { gpr_refcount refcount; gpr_atm shutdown_count; + int min_read_chunk_size; + int max_read_chunk_size; + /* garbage after the last read */ grpc_slice_buffer last_read_buffer; @@ -111,12 +115,16 @@ typedef struct { } grpc_tcp; static void add_to_estimate(grpc_tcp *tcp, size_t bytes) { - tcp->bytes_read_this_round += bytes; + tcp->bytes_read_this_round += (double)bytes; } static void finish_estimate(grpc_tcp *tcp) { - if (tcp->bytes_read_this_round > tcp->target_length) { - tcp->target_length = 1.5 * tcp->bytes_read_this_round; + /* If we read >80% of the target buffer in one read loop, increase the size + of the target buffer to either the amount read, or twice its previous + value */ + if (tcp->bytes_read_this_round > tcp->target_length * 0.8) { + tcp->target_length = + GPR_MAX(2 * tcp->target_length, tcp->bytes_read_this_round); } else { tcp->target_length = 0.99 * tcp->target_length + 0.01 * tcp->bytes_read_this_round; @@ -129,7 +137,9 @@ static size_t get_target_read_size(grpc_tcp *tcp) { grpc_resource_user_quota(tcp->resource_user)); double target = tcp->target_length * (pressure > 0.8 ? (1.0 - pressure) / 0.2 : 1.0); - return (((size_t)GPR_CLAMP(target, 1024, 4 * 1024 * 1024)) + 255) & + return (((size_t)GPR_CLAMP(target, tcp->min_read_chunk_size, + tcp->max_read_chunk_size)) + + 255) & ~(size_t)255; } @@ -562,9 +572,50 @@ static const grpc_endpoint_vtable vtable = {tcp_read, tcp_get_peer, tcp_get_fd}; -grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, - grpc_resource_quota *resource_quota, - size_t slice_size, const char *peer_string) { +#define MAX_CHUNK_SIZE 32 * 1024 * 1024 + +grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_fd *em_fd, + const grpc_channel_args *channel_args, + const char *peer_string) { + int tcp_read_chunk_size = GRPC_TCP_DEFAULT_READ_SLICE_SIZE; + int tcp_max_read_chunk_size = 4 * 1024 * 1024; + int tcp_min_read_chunk_size = 256; + grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); + if (channel_args != NULL) { + for (size_t i = 0; i < channel_args->num_args; i++) { + if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_TCP_READ_CHUNK_SIZE)) { + grpc_integer_options options = {(int)tcp_read_chunk_size, 1, + MAX_CHUNK_SIZE}; + tcp_read_chunk_size = + grpc_channel_arg_get_integer(&channel_args->args[i], options); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_TCP_MIN_READ_CHUNK_SIZE)) { + grpc_integer_options options = {(int)tcp_read_chunk_size, 1, + MAX_CHUNK_SIZE}; + tcp_min_read_chunk_size = + grpc_channel_arg_get_integer(&channel_args->args[i], options); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_TCP_MAX_READ_CHUNK_SIZE)) { + grpc_integer_options options = {(int)tcp_read_chunk_size, 1, + MAX_CHUNK_SIZE}; + tcp_max_read_chunk_size = + grpc_channel_arg_get_integer(&channel_args->args[i], options); + } else if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { + grpc_resource_quota_unref_internal(exec_ctx, resource_quota); + resource_quota = grpc_resource_quota_ref_internal( + channel_args->args[i].value.pointer.p); + } + } + } + + if (tcp_min_read_chunk_size > tcp_max_read_chunk_size) { + tcp_min_read_chunk_size = tcp_max_read_chunk_size; + } + tcp_read_chunk_size = GPR_CLAMP(tcp_read_chunk_size, tcp_min_read_chunk_size, + tcp_max_read_chunk_size); + grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); tcp->base.vtable = &vtable; tcp->peer_string = gpr_strdup(peer_string); @@ -574,7 +625,9 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, tcp->release_fd_cb = NULL; tcp->release_fd = NULL; tcp->incoming_buffer = NULL; - tcp->target_length = slice_size; + tcp->target_length = (double)tcp_read_chunk_size; + tcp->min_read_chunk_size = tcp_min_read_chunk_size; + tcp->max_read_chunk_size = tcp_max_read_chunk_size; tcp->bytes_read_this_round = 0; tcp->iov_size = 1; tcp->finished_edge = true; @@ -592,6 +645,7 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, &tcp->slice_allocator, tcp->resource_user, tcp_read_allocation_done, tcp); /* Tell network status tracker about new endpoint */ grpc_network_status_register_endpoint(&tcp->base); + grpc_resource_quota_unref_internal(exec_ctx, resource_quota); return &tcp->base; } diff --git a/src/core/lib/iomgr/tcp_posix.h b/src/core/lib/iomgr/tcp_posix.h index 1c0d13f96e2..3bf93dfcf85 100644 --- a/src/core/lib/iomgr/tcp_posix.h +++ b/src/core/lib/iomgr/tcp_posix.h @@ -49,12 +49,21 @@ #define GRPC_TCP_DEFAULT_READ_SLICE_SIZE 8192 +/* Channel arg (integer) setting how large a slice to try and read from the wire + each time recvmsg (or equivalent) is called */ +#define GRPC_ARG_TCP_READ_CHUNK_SIZE "grpc.experimental.tcp_read_chunk_size" +#define GRPC_ARG_TCP_MIN_READ_CHUNK_SIZE \ + "grpc.experimental.tcp_min_read_chunk_size" +#define GRPC_ARG_TCP_MAX_READ_CHUNK_SIZE \ + "grpc.experimental.tcp_max_read_chunk_size" + extern int grpc_tcp_trace; /* Create a tcp endpoint given a file desciptor and a read slice size. Takes ownership of fd. */ -grpc_endpoint *grpc_tcp_create(grpc_fd *fd, grpc_resource_quota *resource_quota, - size_t read_slice_size, const char *peer_string); +grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_fd *fd, + const grpc_channel_args *args, + const char *peer_string); /* Return the tcp endpoint's fd, or -1 if this is not available. Does not release the fd. diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c index d6a017cf7f6..e66ffc9b1c2 100644 --- a/src/core/lib/iomgr/tcp_server_posix.c +++ b/src/core/lib/iomgr/tcp_server_posix.c @@ -59,6 +59,7 @@ #include #include +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/iomgr/sockaddr_utils.h" @@ -90,7 +91,6 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s = gpr_zalloc(sizeof(grpc_tcp_server)); s->so_reuseport = has_so_reuseport; - s->resource_quota = grpc_resource_quota_create(NULL); s->expand_wildcard_addrs = false; for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) { if (0 == strcmp(GRPC_ARG_ALLOW_REUSEPORT, args->args[i].key)) { @@ -98,27 +98,14 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, s->so_reuseport = has_so_reuseport && (args->args[i].value.integer != 0); } else { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); gpr_free(s); return GRPC_ERROR_CREATE_FROM_STATIC_STRING(GRPC_ARG_ALLOW_REUSEPORT " must be an integer"); } - } else if (0 == strcmp(GRPC_ARG_RESOURCE_QUOTA, args->args[i].key)) { - if (args->args[i].type == GRPC_ARG_POINTER) { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); - s->resource_quota = - grpc_resource_quota_ref_internal(args->args[i].value.pointer.p); - } else { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); - gpr_free(s); - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - GRPC_ARG_RESOURCE_QUOTA " must be a pointer to a buffer pool"); - } } else if (0 == strcmp(GRPC_ARG_EXPAND_WILDCARD_ADDRS, args->args[i].key)) { if (args->args[i].type == GRPC_ARG_INTEGER) { s->expand_wildcard_addrs = (args->args[i].value.integer != 0); } else { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); gpr_free(s); return GRPC_ERROR_CREATE_FROM_STATIC_STRING( GRPC_ARG_EXPAND_WILDCARD_ADDRS " must be an integer"); @@ -138,6 +125,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, s->head = NULL; s->tail = NULL; s->nports = 0; + s->channel_args = grpc_channel_args_copy(args); gpr_atm_no_barrier_store(&s->next_pollset_to_assign, 0); *server = s; return GRPC_ERROR_NONE; @@ -158,8 +146,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) { s->head = sp->next; gpr_free(sp); } - - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); + grpc_channel_args_destroy(exec_ctx, s->channel_args); gpr_free(s); } @@ -286,8 +273,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) { sp->server->on_accept_cb( exec_ctx, sp->server->on_accept_cb_arg, - grpc_tcp_create(fdobj, sp->server->resource_quota, - GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str), + grpc_tcp_create(exec_ctx, fdobj, sp->server->channel_args, addr_str), read_notifier_pollset, acceptor); gpr_free(name); diff --git a/src/core/lib/iomgr/tcp_server_utils_posix.h b/src/core/lib/iomgr/tcp_server_utils_posix.h index f5dc8532f9f..c15e2e1493a 100644 --- a/src/core/lib/iomgr/tcp_server_utils_posix.h +++ b/src/core/lib/iomgr/tcp_server_utils_posix.h @@ -103,7 +103,8 @@ struct grpc_tcp_server { /* next pollset to assign a channel to */ gpr_atm next_pollset_to_assign; - grpc_resource_quota *resource_quota; + /* channel args for this server */ + grpc_channel_args *channel_args; }; /* If successful, add a listener to \a s for \a addr, set \a dsmode for the diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c index 4870dc1a536..9a566e6484c 100644 --- a/test/core/bad_client/bad_client.c +++ b/test/core/bad_client/bad_client.c @@ -117,10 +117,7 @@ void grpc_run_bad_client_test( grpc_init(); /* Create endpoints */ - grpc_resource_quota *resource_quota = - grpc_resource_quota_create("bad_client_test"); - sfd = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, 65536); - grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); + sfd = grpc_iomgr_create_endpoint_pair("fixture", NULL); /* Create server, completion events */ a.server = grpc_server_create(NULL, NULL); diff --git a/test/core/end2end/fixtures/h2_sockpair+trace.c b/test/core/end2end/fixtures/h2_sockpair+trace.c index edf42a40702..9b4526cb75c 100644 --- a/test/core/end2end/fixtures/h2_sockpair+trace.c +++ b/test/core/end2end/fixtures/h2_sockpair+trace.c @@ -96,9 +96,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_socketpair( f.fixture_data = sfd; f.cq = grpc_completion_queue_create(NULL); - grpc_resource_quota *resource_quota = grpc_resource_quota_create("fixture"); - *sfd = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, 65536); - grpc_resource_quota_unref(resource_quota); + *sfd = grpc_iomgr_create_endpoint_pair("fixture", NULL); return f; } diff --git a/test/core/end2end/fixtures/h2_sockpair.c b/test/core/end2end/fixtures/h2_sockpair.c index 94b2623b3ea..3aaa0ce6fb3 100644 --- a/test/core/end2end/fixtures/h2_sockpair.c +++ b/test/core/end2end/fixtures/h2_sockpair.c @@ -90,9 +90,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_socketpair( f.fixture_data = sfd; f.cq = grpc_completion_queue_create(NULL); - grpc_resource_quota *resource_quota = grpc_resource_quota_create("fixture"); - *sfd = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, 65536); - grpc_resource_quota_unref(resource_quota); + *sfd = grpc_iomgr_create_endpoint_pair("fixture", NULL); return f; } diff --git a/test/core/end2end/fixtures/h2_sockpair_1byte.c b/test/core/end2end/fixtures/h2_sockpair_1byte.c index 6f9cf8fe26a..87410dd1e98 100644 --- a/test/core/end2end/fixtures/h2_sockpair_1byte.c +++ b/test/core/end2end/fixtures/h2_sockpair_1byte.c @@ -48,6 +48,7 @@ #include "src/core/lib/channel/http_server_filter.h" #include "src/core/lib/iomgr/endpoint_pair.h" #include "src/core/lib/iomgr/iomgr.h" +#include "src/core/lib/iomgr/tcp_posix.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/server.h" @@ -90,9 +91,17 @@ static grpc_end2end_test_fixture chttp2_create_fixture_socketpair( f.fixture_data = sfd; f.cq = grpc_completion_queue_create(NULL); - grpc_resource_quota *resource_quota = grpc_resource_quota_create("fixture"); - *sfd = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, 1); - grpc_resource_quota_unref(resource_quota); + grpc_arg a[] = {{.key = GRPC_ARG_TCP_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = 1}, + {.key = GRPC_ARG_TCP_MIN_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = 1}, + {.key = GRPC_ARG_TCP_MAX_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = 1}}; + grpc_channel_args args = {.num_args = GPR_ARRAY_SIZE(a), .args = a}; + *sfd = grpc_iomgr_create_endpoint_pair("fixture", &args); return f; } diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c index 4b98ef257e5..491b05f41fd 100644 --- a/test/core/iomgr/endpoint_pair_test.c +++ b/test/core/iomgr/endpoint_pair_test.c @@ -37,6 +37,7 @@ #include #include #include +#include "src/core/lib/iomgr/tcp_posix.h" #include "test/core/iomgr/endpoint_tests.h" #include "test/core/util/test_config.h" @@ -49,11 +50,11 @@ static grpc_endpoint_test_fixture create_fixture_endpoint_pair( size_t slice_size) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_endpoint_test_fixture f; - grpc_resource_quota *resource_quota = - grpc_resource_quota_create("endpoint_pair_test"); - grpc_endpoint_pair p = - grpc_iomgr_create_endpoint_pair("test", resource_quota, slice_size); - grpc_resource_quota_unref(resource_quota); + grpc_arg a[] = {{.key = GRPC_ARG_TCP_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = (int)slice_size}}; + grpc_channel_args args = {.num_args = GPR_ARRAY_SIZE(a), .args = a}; + grpc_endpoint_pair p = grpc_iomgr_create_endpoint_pair("test", &args); f.client_ep = p.client; f.server_ep = p.server; diff --git a/test/core/iomgr/fd_conservation_posix_test.c b/test/core/iomgr/fd_conservation_posix_test.c index 3dffa02c3c8..6ac322bb014 100644 --- a/test/core/iomgr/fd_conservation_posix_test.c +++ b/test/core/iomgr/fd_conservation_posix_test.c @@ -57,7 +57,7 @@ int main(int argc, char **argv) { for (i = 0; i < 100; i++) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - p = grpc_iomgr_create_endpoint_pair("test", resource_quota, 1); + p = grpc_iomgr_create_endpoint_pair("test", NULL); grpc_endpoint_destroy(&exec_ctx, p.client); grpc_endpoint_destroy(&exec_ctx, p.server); grpc_exec_ctx_finish(&exec_ctx); diff --git a/test/core/iomgr/tcp_posix_test.c b/test/core/iomgr/tcp_posix_test.c index 5a55be888fe..2c53a003d23 100644 --- a/test/core/iomgr/tcp_posix_test.c +++ b/test/core/iomgr/tcp_posix_test.c @@ -183,10 +183,12 @@ static void read_test(size_t num_bytes, size_t slice_size) { create_sockets(sv); - grpc_resource_quota *resource_quota = grpc_resource_quota_create("read_test"); - ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), resource_quota, - slice_size, "test"); - grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); + grpc_arg a[] = {{.key = GRPC_ARG_TCP_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = (int)slice_size}}; + grpc_channel_args args = {.num_args = GPR_ARRAY_SIZE(a), .args = a}; + ep = grpc_tcp_create(&exec_ctx, grpc_fd_create(sv[1], "read_test"), &args, + "test"); grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); @@ -233,11 +235,12 @@ static void large_read_test(size_t slice_size) { create_sockets(sv); - grpc_resource_quota *resource_quota = - grpc_resource_quota_create("large_read_test"); - ep = grpc_tcp_create(grpc_fd_create(sv[1], "large_read_test"), resource_quota, - slice_size, "test"); - grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); + grpc_arg a[] = {{.key = GRPC_ARG_TCP_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = (int)slice_size}}; + grpc_channel_args args = {.num_args = GPR_ARRAY_SIZE(a), .args = a}; + ep = grpc_tcp_create(&exec_ctx, grpc_fd_create(sv[1], "large_read_test"), + &args, "test"); grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket(sv[0]); @@ -372,11 +375,12 @@ static void write_test(size_t num_bytes, size_t slice_size) { create_sockets(sv); - grpc_resource_quota *resource_quota = - grpc_resource_quota_create("write_test"); - ep = grpc_tcp_create(grpc_fd_create(sv[1], "write_test"), resource_quota, - GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "test"); - grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); + grpc_arg a[] = {{.key = GRPC_ARG_TCP_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = (int)slice_size}}; + grpc_channel_args args = {.num_args = GPR_ARRAY_SIZE(a), .args = a}; + ep = grpc_tcp_create(&exec_ctx, grpc_fd_create(sv[1], "write_test"), &args, + "test"); grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); state.ep = ep; @@ -441,12 +445,13 @@ static void release_fd_test(size_t num_bytes, size_t slice_size) { create_sockets(sv); - grpc_resource_quota *resource_quota = - grpc_resource_quota_create("release_fd_test"); - ep = grpc_tcp_create(grpc_fd_create(sv[1], "read_test"), resource_quota, - slice_size, "test"); + grpc_arg a[] = {{.key = GRPC_ARG_TCP_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = (int)slice_size}}; + grpc_channel_args args = {.num_args = GPR_ARRAY_SIZE(a), .args = a}; + ep = grpc_tcp_create(&exec_ctx, grpc_fd_create(sv[1], "read_test"), &args, + "test"); GPR_ASSERT(grpc_tcp_fd(ep) == sv[1] && sv[1] >= 0); - grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); grpc_endpoint_add_to_pollset(&exec_ctx, ep, g_pollset); written_bytes = fill_socket_partial(sv[0], num_bytes); @@ -534,10 +539,14 @@ static grpc_endpoint_test_fixture create_fixture_tcp_socketpair( create_sockets(sv); grpc_resource_quota *resource_quota = grpc_resource_quota_create("tcp_posix_test_socketpair"); - f.client_ep = grpc_tcp_create(grpc_fd_create(sv[0], "fixture:client"), - resource_quota, slice_size, "test"); - f.server_ep = grpc_tcp_create(grpc_fd_create(sv[1], "fixture:server"), - resource_quota, slice_size, "test"); + grpc_arg a[] = {{.key = GRPC_ARG_TCP_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = (int)slice_size}}; + grpc_channel_args args = {.num_args = GPR_ARRAY_SIZE(a), .args = a}; + f.client_ep = grpc_tcp_create( + &exec_ctx, grpc_fd_create(sv[0], "fixture:client"), &args, "test"); + f.server_ep = grpc_tcp_create( + &exec_ctx, grpc_fd_create(sv[1], "fixture:server"), &args, "test"); grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset); grpc_endpoint_add_to_pollset(&exec_ctx, f.server_ep, g_pollset); diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index b3775e91a72..e6e3d91be5d 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -39,8 +39,10 @@ #include #include #include +#include #include "src/core/lib/iomgr/endpoint_pair.h" #include "src/core/lib/iomgr/iomgr.h" +#include "src/core/lib/iomgr/tcp_posix.h" #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/tsi/fake_transport_security.h" @@ -57,10 +59,11 @@ static grpc_endpoint_test_fixture secure_endpoint_create_fixture_tcp_socketpair( grpc_endpoint_test_fixture f; grpc_endpoint_pair tcp; - grpc_resource_quota *resource_quota = - grpc_resource_quota_create("secure_endpoint_test"); - tcp = grpc_iomgr_create_endpoint_pair("fixture", resource_quota, slice_size); - grpc_resource_quota_unref_internal(&exec_ctx, resource_quota); + grpc_arg a[] = {{.key = GRPC_ARG_TCP_READ_CHUNK_SIZE, + .type = GRPC_ARG_INTEGER, + .value.integer = (int)slice_size}}; + grpc_channel_args args = {.num_args = GPR_ARRAY_SIZE(a), .args = a}; + tcp = grpc_iomgr_create_endpoint_pair("fixture", &args); grpc_endpoint_add_to_pollset(&exec_ctx, tcp.client, g_pollset); grpc_endpoint_add_to_pollset(&exec_ctx, tcp.server, g_pollset); From c9a3aaaa541f6915c0ea886acc1026eba27da881 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 28 Mar 2017 15:50:25 -0700 Subject: [PATCH 25/90] Fix Python poll() server spinlock bug --- src/python/grpcio/grpc/_server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/python/grpcio/grpc/_server.py b/src/python/grpcio/grpc/_server.py index 47838c2c986..f29c44a4cfd 100644 --- a/src/python/grpcio/grpc/_server.py +++ b/src/python/grpcio/grpc/_server.py @@ -705,6 +705,10 @@ def _serve(state): state.rpc_states.remove(rpc_state) if _stop_serving(state): return + # We want to force the deletion of the previous event + # ~before~ we poll again; if the event has a reference + # to a shutdown Call object, this can induce spinlock. + event = None def _stop(state, grace): From 9f4133080cebd9a9a6bbe00df5c20e6b29f79f53 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Mar 2017 16:42:49 -0700 Subject: [PATCH 26/90] Allow resource quota overcommit --- src/core/lib/iomgr/resource_quota.c | 35 ++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 511ffdcdf13..3a0fd5d81ac 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -246,7 +246,8 @@ static void rulist_remove(grpc_resource_user *resource_user, grpc_rulist list) { */ static bool rq_alloc(grpc_exec_ctx *exec_ctx, - grpc_resource_quota *resource_quota); + grpc_resource_quota *resource_quota, + bool check_memory_usage); static bool rq_reclaim_from_per_user_free_pool( grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota); static bool rq_reclaim(grpc_exec_ctx *exec_ctx, @@ -256,11 +257,20 @@ static void rq_step(grpc_exec_ctx *exec_ctx, void *rq, grpc_error *error) { grpc_resource_quota *resource_quota = rq; resource_quota->step_scheduled = false; do { - if (rq_alloc(exec_ctx, resource_quota)) goto done; + if (rq_alloc(exec_ctx, resource_quota, false)) goto done; } while (rq_reclaim_from_per_user_free_pool(exec_ctx, resource_quota)); if (!rq_reclaim(exec_ctx, resource_quota, false)) { - rq_reclaim(exec_ctx, resource_quota, true); + if (!rq_reclaim(exec_ctx, resource_quota, true)) { + /* no reclaimation possible, but we've still got allocation requests: + start satisfying them without regard to memory utilization to avoid + infinitely stuck requests */ + gpr_log(GPR_DEBUG, + "ResourceQuota WARNING: allocating requests without free pool " + "available"); + while (!rq_alloc(exec_ctx, resource_quota, true)) + ; + } } done: @@ -286,15 +296,22 @@ static void rq_update_estimate(grpc_resource_quota *resource_quota) { MEMORY_USAGE_ESTIMATION_MAX)); } +static bool fits_pool(int64_t want, int64_t have, bool check) { + if (!check) return true; + return want <= have; +} + /* returns true if all allocations are completed */ static bool rq_alloc(grpc_exec_ctx *exec_ctx, - grpc_resource_quota *resource_quota) { + grpc_resource_quota *resource_quota, + bool check_memory_usage) { grpc_resource_user *resource_user; while ((resource_user = rulist_pop_head(resource_quota, GRPC_RULIST_AWAITING_ALLOCATION))) { gpr_mu_lock(&resource_user->mu); if (resource_user->free_pool < 0 && - -resource_user->free_pool <= resource_quota->free_pool) { + fits_pool(-resource_user->free_pool, resource_quota->free_pool, + check_memory_usage)) { int64_t amt = -resource_user->free_pool; resource_user->free_pool = 0; resource_quota->free_pool -= amt; @@ -314,6 +331,11 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx, grpc_closure_list_sched(exec_ctx, &resource_user->on_allocated); gpr_mu_unlock(&resource_user->mu); } else { + gpr_log(GPR_DEBUG, "RQ %s %s: delay alloc %" PRId64 + " bytes; rq_free_pool = %" PRId64, + resource_quota->name, resource_user->name, + -resource_user->free_pool, resource_quota->free_pool); + GPR_ASSERT(check_memory_usage); rulist_add_head(resource_user, GRPC_RULIST_AWAITING_ALLOCATION); gpr_mu_unlock(&resource_user->mu); return false; @@ -333,6 +355,9 @@ static bool rq_reclaim_from_per_user_free_pool( int64_t amt = resource_user->free_pool; resource_user->free_pool = 0; resource_quota->free_pool += amt; + if (resource_quota->free_pool > resource_quota->size) { + resource_quota->free_pool = resource_quota->size; + } rq_update_estimate(resource_quota); if (grpc_resource_quota_trace) { gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64 From 484871bb03dc877a0bb84bbeab7614a1419d6e08 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 28 Mar 2017 18:00:33 -0700 Subject: [PATCH 27/90] Bump version to 1.2.1 --- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Auth/project.json | 4 ++-- src/csharp/Grpc.Core.Testing/project.json | 4 ++-- src/csharp/Grpc.Core/VersionInfo.cs | 2 +- src/csharp/Grpc.Core/project.json | 2 +- src/csharp/Grpc.HealthCheck/project.json | 4 ++-- src/csharp/Grpc.Reflection/project.json | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 31 files changed, 39 insertions(+), 39 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1543140c0aa..f9ca86266bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.2.1-pre2") +set(PACKAGE_VERSION "1.2.1") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 041e4f5c7c5..5655ae36858 100644 --- a/Makefile +++ b/Makefile @@ -412,8 +412,8 @@ Q = @ endif CORE_VERSION = 3.0.0-dev -CPP_VERSION = 1.2.1-pre2 -CSHARP_VERSION = 1.2.1-pre2 +CPP_VERSION = 1.2.1 +CSHARP_VERSION = 1.2.1 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 49d64bd1da0..1c3a23bf6b7 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 3.0.0-dev g_stands_for: green - version: 1.2.1-pre2 + version: 1.2.1 filegroups: - name: census public_headers: diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index fe1f3e008b8..585713c0e61 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -37,7 +37,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.2.1-pre2' + version = '1.2.1' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index b5c172ee82e..cc9817a02a9 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.2.1-pre2' + version = '1.2.1' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 6c2736cd78e..e631ee5104a 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.2.1-pre2' + version = '1.2.1' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'http://www.grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 7031b3d1449..ea78d5f91ea 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -35,7 +35,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.2.1-pre2' + version = '1.2.1' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' diff --git a/package.json b/package.json index e75dfa88206..31846bf2e79 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.2.1-pre2", + "version": "1.2.1", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "http://www.grpc.io/", diff --git a/package.xml b/package.xml index 44a15c23a96..24a0bf01a8b 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2017-03-01 - 1.2.1RC2 - 1.2.1RC2 + 1.2.1 + 1.2.1 beta diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 593e17e7706..5cac8cdf781 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -37,5 +37,5 @@ #include namespace grpc { -grpc::string Version() { return "1.2.1-pre2"; } +grpc::string Version() { return "1.2.1"; } } diff --git a/src/csharp/Grpc.Auth/project.json b/src/csharp/Grpc.Auth/project.json index 6345946c39c..66b3fa93d6a 100644 --- a/src/csharp/Grpc.Auth/project.json +++ b/src/csharp/Grpc.Auth/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre2", + "version": "1.2.1", "title": "gRPC C# Auth", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1-pre2", + "Grpc.Core": "1.2.1", "Google.Apis.Auth": "1.21.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Core.Testing/project.json b/src/csharp/Grpc.Core.Testing/project.json index 7337e2d15a8..c4e015f4186 100644 --- a/src/csharp/Grpc.Core.Testing/project.json +++ b/src/csharp/Grpc.Core.Testing/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre2", + "version": "1.2.1", "title": "gRPC C# Core Testing", "authors": [ "Google Inc." ], "copyright": "Copyright 2017, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1-pre2" + "Grpc.Core": "1.2.1" }, "frameworks": { "net45": { diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 976508dfac2..42881c9f065 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -53,6 +53,6 @@ namespace Grpc.Core /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.2.1-pre2"; + public const string CurrentVersion = "1.2.1"; } } diff --git a/src/csharp/Grpc.Core/project.json b/src/csharp/Grpc.Core/project.json index d0eb7759782..f9f617772a8 100644 --- a/src/csharp/Grpc.Core/project.json +++ b/src/csharp/Grpc.Core/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre2", + "version": "1.2.1", "title": "gRPC C# Core", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", diff --git a/src/csharp/Grpc.HealthCheck/project.json b/src/csharp/Grpc.HealthCheck/project.json index ff5429dba6b..0a564fde5af 100644 --- a/src/csharp/Grpc.HealthCheck/project.json +++ b/src/csharp/Grpc.HealthCheck/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre2", + "version": "1.2.1", "title": "gRPC C# Healthchecking", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1-pre2", + "Grpc.Core": "1.2.1", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Reflection/project.json b/src/csharp/Grpc.Reflection/project.json index ac7696e9074..ac948d45f04 100644 --- a/src/csharp/Grpc.Reflection/project.json +++ b/src/csharp/Grpc.Reflection/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1-pre2", + "version": "1.2.1", "title": "gRPC C# Reflection", "authors": [ "Google Inc." ], "copyright": "Copyright 2016, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1-pre2", + "Grpc.Core": "1.2.1", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index 8a6f22f1fa3..da6556dc27e 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -28,7 +28,7 @@ @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @rem Current package versions -set VERSION=1.2.1-pre2 +set VERSION=1.2.1 set PROTOBUF_VERSION=3.0.0 @rem Adjust the location of nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 56e24de1993..c0d58d667a9 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -66,7 +66,7 @@ dotnet pack --configuration Release Grpc.Auth/project.json --output ../../artifa dotnet pack --configuration Release Grpc.HealthCheck/project.json --output ../../artifacts dotnet pack --configuration Release Grpc.Reflection/project.json --output ../../artifacts -nuget pack Grpc.nuspec -Version "1.2.1-pre2" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.2.1-pre2" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.2.1" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.2.1" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 630b3cba006..177b27b9112 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.2.1-pre2", + "version": "1.2.1", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.2.1-pre2", + "grpc": "^1.2.1", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index 98863c1d776..5867db6bb9b 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.2.1-pre2", + "version": "1.2.1", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "http://www.grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 636b9f792aa..226e36fd866 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.2.1-pre2' + v = '1.2.1' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 9339adce431..60866a078af 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -38,4 +38,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.2.1-pre2" +#define GRPC_OBJC_VERSION_STRING @"1.2.1" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 418ca993bc7..330c20dd8f3 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.2.1rc2' +VERSION='1.2.1' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index 00b7680a0f5..c06086d5b8c 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.2.1rc2' +VERSION='1.2.1' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index cf42d123ae3..f774e403f50 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.2.1rc2' +VERSION='1.2.1' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index acd2b85c95e..9a1a472afcc 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.2.1rc2' +VERSION='1.2.1' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 7d54034a298..3ee48ae328f 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -29,5 +29,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.2.1.pre2' + VERSION = '1.2.1' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index d6a7be76efa..d037014852f 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -29,6 +29,6 @@ module GRPC module Tools - VERSION = '1.2.1.pre2' + VERSION = '1.2.1' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index fede4440800..e125d189220 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.2.1rc2' +VERSION='1.2.1' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 13734e8d76a..4d427640503 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.1-pre2 +PROJECT_NUMBER = 1.2.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index dfef610a00b..8e5730a0d32 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.1-pre2 +PROJECT_NUMBER = 1.2.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From 72240e1c1b42c32eb40de8803a120a8c72ec8e43 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Tue, 28 Mar 2017 18:19:46 -0700 Subject: [PATCH 28/90] Update BUILD file --- BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD b/BUILD index faaef5198ab..0c6daee4bb8 100644 --- a/BUILD +++ b/BUILD @@ -41,7 +41,7 @@ g_stands_for = "green" core_version = "3.0.0-dev" -version = "1.2.1-pre2" +version = "1.2.1" grpc_cc_library( name = "gpr", From d89bf42e205c2011127d46de399400fb55d9916a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 09:05:36 -0700 Subject: [PATCH 29/90] Revert "Allow resource quota overcommit" This reverts commit 9f4133080cebd9a9a6bbe00df5c20e6b29f79f53. --- src/core/lib/iomgr/resource_quota.c | 35 +++++------------------------ 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 3a0fd5d81ac..511ffdcdf13 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -246,8 +246,7 @@ static void rulist_remove(grpc_resource_user *resource_user, grpc_rulist list) { */ static bool rq_alloc(grpc_exec_ctx *exec_ctx, - grpc_resource_quota *resource_quota, - bool check_memory_usage); + grpc_resource_quota *resource_quota); static bool rq_reclaim_from_per_user_free_pool( grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota); static bool rq_reclaim(grpc_exec_ctx *exec_ctx, @@ -257,20 +256,11 @@ static void rq_step(grpc_exec_ctx *exec_ctx, void *rq, grpc_error *error) { grpc_resource_quota *resource_quota = rq; resource_quota->step_scheduled = false; do { - if (rq_alloc(exec_ctx, resource_quota, false)) goto done; + if (rq_alloc(exec_ctx, resource_quota)) goto done; } while (rq_reclaim_from_per_user_free_pool(exec_ctx, resource_quota)); if (!rq_reclaim(exec_ctx, resource_quota, false)) { - if (!rq_reclaim(exec_ctx, resource_quota, true)) { - /* no reclaimation possible, but we've still got allocation requests: - start satisfying them without regard to memory utilization to avoid - infinitely stuck requests */ - gpr_log(GPR_DEBUG, - "ResourceQuota WARNING: allocating requests without free pool " - "available"); - while (!rq_alloc(exec_ctx, resource_quota, true)) - ; - } + rq_reclaim(exec_ctx, resource_quota, true); } done: @@ -296,22 +286,15 @@ static void rq_update_estimate(grpc_resource_quota *resource_quota) { MEMORY_USAGE_ESTIMATION_MAX)); } -static bool fits_pool(int64_t want, int64_t have, bool check) { - if (!check) return true; - return want <= have; -} - /* returns true if all allocations are completed */ static bool rq_alloc(grpc_exec_ctx *exec_ctx, - grpc_resource_quota *resource_quota, - bool check_memory_usage) { + grpc_resource_quota *resource_quota) { grpc_resource_user *resource_user; while ((resource_user = rulist_pop_head(resource_quota, GRPC_RULIST_AWAITING_ALLOCATION))) { gpr_mu_lock(&resource_user->mu); if (resource_user->free_pool < 0 && - fits_pool(-resource_user->free_pool, resource_quota->free_pool, - check_memory_usage)) { + -resource_user->free_pool <= resource_quota->free_pool) { int64_t amt = -resource_user->free_pool; resource_user->free_pool = 0; resource_quota->free_pool -= amt; @@ -331,11 +314,6 @@ static bool rq_alloc(grpc_exec_ctx *exec_ctx, grpc_closure_list_sched(exec_ctx, &resource_user->on_allocated); gpr_mu_unlock(&resource_user->mu); } else { - gpr_log(GPR_DEBUG, "RQ %s %s: delay alloc %" PRId64 - " bytes; rq_free_pool = %" PRId64, - resource_quota->name, resource_user->name, - -resource_user->free_pool, resource_quota->free_pool); - GPR_ASSERT(check_memory_usage); rulist_add_head(resource_user, GRPC_RULIST_AWAITING_ALLOCATION); gpr_mu_unlock(&resource_user->mu); return false; @@ -355,9 +333,6 @@ static bool rq_reclaim_from_per_user_free_pool( int64_t amt = resource_user->free_pool; resource_user->free_pool = 0; resource_quota->free_pool += amt; - if (resource_quota->free_pool > resource_quota->size) { - resource_quota->free_pool = resource_quota->size; - } rq_update_estimate(resource_quota); if (grpc_resource_quota_trace) { gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64 From b58de7238981dc5fa3a18db4941bddca75c0d521 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 14:15:12 -0700 Subject: [PATCH 30/90] Lazily allocate batch control objects --- src/core/lib/surface/call.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 50ddbc92aa4..03b1fbf11aa 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -163,7 +163,7 @@ struct grpc_call { /* have we received initial metadata */ bool has_initial_md_been_received; - batch_control active_batches[MAX_CONCURRENT_BATCHES]; + batch_control *active_batches[MAX_CONCURRENT_BATCHES]; grpc_transport_stream_op_payload stream_op_payload; /* first idx: is_receiving, second idx: is_trailing */ @@ -1022,7 +1022,11 @@ static batch_control *allocate_batch_control(grpc_call *call, const grpc_op *ops, size_t num_ops) { int slot = batch_slot_for_op(ops[0].op); - batch_control *bctl = &call->active_batches[slot]; + batch_control **pslot = &call->active_batches[slot]; + if (*pslot == NULL) { + *pslot = gpr_arena_alloc(call->arena, sizeof(batch_control)); + } + batch_control *bctl = *pslot; if (bctl->call != NULL) { return NULL; } From 1c10a7b3b4330244ab038ab6985247ffb4a62836 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Mar 2017 14:35:16 -0700 Subject: [PATCH 31/90] Lazily track parenting --- src/core/lib/surface/call.c | 135 ++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 51 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 50ddbc92aa4..946ad1b83ed 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -133,16 +133,28 @@ typedef struct batch_control { grpc_transport_stream_op op; } batch_control; +typedef struct { + gpr_mu child_list_mu; + grpc_call *first_child; +} parent_call; + +typedef struct { + grpc_call *parent; + /** siblings: children of the same parent form a list, and this list is + protected under + parent->mu */ + grpc_call *sibling_next; + grpc_call *sibling_prev; +} child_call; + struct grpc_call { gpr_arena *arena; grpc_completion_queue *cq; grpc_polling_entity pollent; grpc_channel *channel; - grpc_call *parent; - grpc_call *first_child; gpr_timespec start_time; - /* protects first_child, and child next/prev links */ - gpr_mu child_list_mu; + /* parent_call* */ gpr_atm parent_call_atm; + child_call *child_call; /* client or server call */ bool is_client; @@ -194,12 +206,6 @@ struct grpc_call { int send_extra_metadata_count; gpr_timespec send_deadline; - /** siblings: children of the same parent form a list, and this list is - protected under - parent->mu */ - grpc_call *sibling_next; - grpc_call *sibling_prev; - grpc_slice_buffer_stream sending_stream; grpc_byte_stream *receiving_stream; @@ -264,6 +270,23 @@ static void add_init_error(grpc_error **composite, grpc_error *new) { *composite = grpc_error_add_child(*composite, new); } +static parent_call *get_or_create_parent_call(grpc_call *call) { + parent_call *p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); + if (p == NULL) { + p = gpr_arena_alloc(call->arena, sizeof(*p)); + gpr_mu_init(&p->child_list_mu); + if (!gpr_atm_rel_cas(&call->parent_call_atm, (gpr_atm)NULL, (gpr_atm)p)) { + gpr_mu_destroy(&p->child_list_mu); + p = (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); + } + } + return p; +} + +static parent_call *get_parent_call(grpc_call *call) { + return (parent_call *)gpr_atm_acq_load(&call->parent_call_atm); +} + grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, const grpc_call_create_args *args, grpc_call **out_call) { @@ -279,10 +302,8 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, sizeof(grpc_call) + channel_stack->call_stack_size); call->arena = arena; *out_call = call; - gpr_mu_init(&call->child_list_mu); call->channel = args->channel; call->cq = args->cq; - call->parent = args->parent_call; call->start_time = gpr_now(GPR_CLOCK_MONOTONIC); /* Always support no compression */ GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_COMPRESS_NONE); @@ -314,11 +335,17 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, gpr_convert_clock_type(args->send_deadline, GPR_CLOCK_MONOTONIC); if (args->parent_call != NULL) { + child_call *cc = call->child_call = + gpr_arena_alloc(arena, sizeof(child_call)); + call->child_call->parent = args->parent_call; + GRPC_CALL_INTERNAL_REF(args->parent_call, "child"); GPR_ASSERT(call->is_client); GPR_ASSERT(!args->parent_call->is_client); - gpr_mu_lock(&args->parent_call->child_list_mu); + parent_call *pc = get_or_create_parent_call(args->parent_call); + + gpr_mu_lock(&pc->child_list_mu); if (args->propagation_mask & GRPC_PROPAGATE_DEADLINE) { send_deadline = gpr_time_min( @@ -352,17 +379,17 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx, } } - if (args->parent_call->first_child == NULL) { - args->parent_call->first_child = call; - call->sibling_next = call->sibling_prev = call; + if (pc->first_child == NULL) { + pc->first_child = call; + cc->sibling_next = cc->sibling_prev = call; } else { - call->sibling_next = args->parent_call->first_child; - call->sibling_prev = args->parent_call->first_child->sibling_prev; - call->sibling_next->sibling_prev = call->sibling_prev->sibling_next = - call; + cc->sibling_next = pc->first_child; + cc->sibling_prev = pc->first_child->child_call->sibling_prev; + cc->sibling_next->child_call->sibling_prev = + cc->sibling_prev->child_call->sibling_next = call; } - gpr_mu_unlock(&args->parent_call->child_list_mu); + gpr_mu_unlock(&pc->child_list_mu); } call->send_deadline = send_deadline; @@ -457,7 +484,10 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, if (c->receiving_stream != NULL) { grpc_byte_stream_destroy(exec_ctx, c->receiving_stream); } - gpr_mu_destroy(&c->child_list_mu); + parent_call *pc = get_parent_call(c); + if (pc != NULL) { + gpr_mu_destroy(&pc->child_list_mu); + } for (ii = 0; ii < c->send_extra_metadata_count; ii++) { GRPC_MDELEM_UNREF(exec_ctx, c->send_extra_metadata[ii].md); } @@ -487,31 +517,31 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, } void grpc_call_destroy(grpc_call *c) { - int cancel; - grpc_call *parent = c->parent; + parent_call *pc = get_parent_call(c); + child_call *cc = c->child_call; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_call_destroy", 0); GRPC_API_TRACE("grpc_call_destroy(c=%p)", 1, (c)); - if (parent) { - gpr_mu_lock(&parent->child_list_mu); - if (c == parent->first_child) { - parent->first_child = c->sibling_next; - if (c == parent->first_child) { - parent->first_child = NULL; + if (pc) { + gpr_mu_lock(&pc->child_list_mu); + if (c == pc->first_child) { + pc->first_child = cc->sibling_next; + if (c == pc->first_child) { + pc->first_child = NULL; } - c->sibling_prev->sibling_next = c->sibling_next; - c->sibling_next->sibling_prev = c->sibling_prev; } - gpr_mu_unlock(&parent->child_list_mu); - GRPC_CALL_INTERNAL_UNREF(&exec_ctx, parent, "child"); + cc->sibling_prev->child_call->sibling_next = cc->sibling_next; + cc->sibling_next->child_call->sibling_prev = cc->sibling_prev; + gpr_mu_unlock(&pc->child_list_mu); + GRPC_CALL_INTERNAL_UNREF(&exec_ctx, cc->parent, "child"); } GPR_ASSERT(!c->destroy_called); c->destroy_called = 1; - cancel = gpr_atm_acq_load(&c->any_ops_sent_atm) && - !gpr_atm_acq_load(&c->received_final_op_atm); + bool cancel = gpr_atm_acq_load(&c->any_ops_sent_atm) != 0 && + gpr_atm_acq_load(&c->received_final_op_atm) == 0; if (cancel) { cancel_with_error(&exec_ctx, c, STATUS_FROM_API_OVERRIDE, GRPC_ERROR_CANCELLED); @@ -1063,7 +1093,6 @@ static grpc_error *consolidate_batch_errors(batch_control *bctl) { static void post_batch_completion(grpc_exec_ctx *exec_ctx, batch_control *bctl) { - grpc_call *child_call; grpc_call *next_child_call; grpc_call *call = bctl->call; grpc_error *error = consolidate_batch_errors(bctl); @@ -1088,21 +1117,25 @@ static void post_batch_completion(grpc_exec_ctx *exec_ctx, /* propagate cancellation to any interested children */ gpr_atm_rel_store(&call->received_final_op_atm, 1); - gpr_mu_lock(&call->child_list_mu); - child_call = call->first_child; - if (child_call != NULL) { - do { - next_child_call = child_call->sibling_next; - if (child_call->cancellation_is_inherited) { - GRPC_CALL_INTERNAL_REF(child_call, "propagate_cancel"); - cancel_with_error(exec_ctx, child_call, STATUS_FROM_API_OVERRIDE, - GRPC_ERROR_CANCELLED); - GRPC_CALL_INTERNAL_UNREF(exec_ctx, child_call, "propagate_cancel"); - } - child_call = next_child_call; - } while (child_call != call->first_child); + parent_call *pc = get_parent_call(call); + if (pc != NULL) { + grpc_call *child; + gpr_mu_lock(&pc->child_list_mu); + child = pc->first_child; + if (child != NULL) { + do { + next_child_call = child->child_call->sibling_next; + if (child->cancellation_is_inherited) { + GRPC_CALL_INTERNAL_REF(child, "propagate_cancel"); + cancel_with_error(exec_ctx, child, STATUS_FROM_API_OVERRIDE, + GRPC_ERROR_CANCELLED); + GRPC_CALL_INTERNAL_UNREF(exec_ctx, child, "propagate_cancel"); + } + child = next_child_call; + } while (child != pc->first_child); + } + gpr_mu_unlock(&pc->child_list_mu); } - gpr_mu_unlock(&call->child_list_mu); if (call->is_client) { get_final_status(call, set_status_value_directly, From 561c703afc0e57bae50878394c7c2d9adb15d0ce Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 30 Mar 2017 20:22:31 +0200 Subject: [PATCH 32/90] fix msvc_static_runtime.cmake --- cmake/msvc_static_runtime.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/msvc_static_runtime.cmake b/cmake/msvc_static_runtime.cmake index 5a31ab3d242..fc6d1d62d32 100644 --- a/cmake/msvc_static_runtime.cmake +++ b/cmake/msvc_static_runtime.cmake @@ -3,6 +3,8 @@ option(gRPC_MSVC_STATIC_RUNTIME "Link with static msvc runtime libraries" OFF) if(gRPC_MSVC_STATIC_RUNTIME) # switch from dynamic to static linking of msvcrt foreach(flag_var + CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) From d9a776d4bbf9e8d3989c818c1acce8c61284289a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 30 Mar 2017 12:37:11 -0700 Subject: [PATCH 33/90] Clamp read sizes based on resource quota --- src/core/lib/iomgr/resource_quota.c | 9 +++++++++ src/core/lib/iomgr/resource_quota.h | 2 ++ src/core/lib/iomgr/tcp_posix.c | 19 +++++++++++++------ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/core/lib/iomgr/resource_quota.c b/src/core/lib/iomgr/resource_quota.c index 8dcd80d0011..c3ee8786517 100644 --- a/src/core/lib/iomgr/resource_quota.c +++ b/src/core/lib/iomgr/resource_quota.c @@ -142,6 +142,8 @@ struct grpc_resource_quota { /* Amount of free memory in the resource quota */ int64_t free_pool; + gpr_atm last_size; + /* Has rq_step been scheduled to occur? */ bool step_scheduled; /* Are we currently reclaiming memory */ @@ -581,6 +583,7 @@ grpc_resource_quota *grpc_resource_quota_create(const char *name) { resource_quota->combiner = grpc_combiner_create(NULL); resource_quota->free_pool = INT64_MAX; resource_quota->size = INT64_MAX; + gpr_atm_no_barrier_store(&resource_quota->last_size, GPR_ATM_MAX); resource_quota->step_scheduled = false; resource_quota->reclaiming = false; gpr_atm_no_barrier_store(&resource_quota->memory_usage_estimation, 0); @@ -643,11 +646,17 @@ void grpc_resource_quota_resize(grpc_resource_quota *resource_quota, rq_resize_args *a = gpr_malloc(sizeof(*a)); a->resource_quota = grpc_resource_quota_ref_internal(resource_quota); a->size = (int64_t)size; + gpr_atm_no_barrier_store(&resource_quota->last_size, + (gpr_atm)GPR_MIN((size_t)GPR_ATM_MAX, size)); grpc_closure_init(&a->closure, rq_resize, a, grpc_schedule_on_exec_ctx); grpc_closure_sched(&exec_ctx, &a->closure, GRPC_ERROR_NONE); grpc_exec_ctx_finish(&exec_ctx); } +size_t grpc_resource_quota_peek_size(grpc_resource_quota *resource_quota) { + return (size_t)gpr_atm_no_barrier_load(&resource_quota->last_size); +} + /******************************************************************************* * grpc_resource_user channel args api */ diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index b9f62cbf83f..6f99be0d512 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -90,6 +90,8 @@ grpc_resource_quota *grpc_resource_quota_from_channel_args( double grpc_resource_quota_get_memory_pressure( grpc_resource_quota *resource_quota); +size_t grpc_resource_quota_peek_size(grpc_resource_quota *resource_quota); + typedef struct grpc_resource_user grpc_resource_user; grpc_resource_user *grpc_resource_user_create( diff --git a/src/core/lib/iomgr/tcp_posix.c b/src/core/lib/iomgr/tcp_posix.c index bfc7de59f20..5f4b38de2b9 100644 --- a/src/core/lib/iomgr/tcp_posix.c +++ b/src/core/lib/iomgr/tcp_posix.c @@ -133,14 +133,21 @@ static void finish_estimate(grpc_tcp *tcp) { } static size_t get_target_read_size(grpc_tcp *tcp) { - double pressure = grpc_resource_quota_get_memory_pressure( - grpc_resource_user_quota(tcp->resource_user)); + grpc_resource_quota *rq = grpc_resource_user_quota(tcp->resource_user); + double pressure = grpc_resource_quota_get_memory_pressure(rq); double target = tcp->target_length * (pressure > 0.8 ? (1.0 - pressure) / 0.2 : 1.0); - return (((size_t)GPR_CLAMP(target, tcp->min_read_chunk_size, - tcp->max_read_chunk_size)) + - 255) & - ~(size_t)255; + size_t sz = (((size_t)GPR_CLAMP(target, tcp->min_read_chunk_size, + tcp->max_read_chunk_size)) + + 255) & + ~(size_t)255; + /* don't use more than 1/16th of the overall resource quota for a single read + * alloc */ + size_t rqmax = grpc_resource_quota_peek_size(rq); + if (sz > rqmax / 16 && rqmax > 1024) { + sz = rqmax / 16; + } + return sz; } static grpc_error *tcp_annotate_error(grpc_error *src_error, grpc_tcp *tcp) { From 609eca0e362764f31b9e17ef2babb957fcdd322c Mon Sep 17 00:00:00 2001 From: Wenbo Zhu Date: Thu, 30 Mar 2017 16:18:14 -0700 Subject: [PATCH 34/90] Update PROTOCOL-WEB.md Clarify the use of U-A header. --- doc/PROTOCOL-WEB.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/PROTOCOL-WEB.md b/doc/PROTOCOL-WEB.md index 5f01af36273..6bb280894ab 100644 --- a/doc/PROTOCOL-WEB.md +++ b/doc/PROTOCOL-WEB.md @@ -83,7 +83,8 @@ in the body. User Agent -* U-A: grpc-web-javascript +* Do NOT use User-Agent header (which is to be set by browsers, by default) +* Use X-User-Agent: grpc-web-javascript/0.1 (follow the same format as specified in [gRPC over HTTP2](http://www.grpc.io/docs/guides/wire.html)) --- From 5e17020d610f743cd7525413856060306f1614be Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 31 Mar 2017 07:30:35 -0700 Subject: [PATCH 35/90] Fix compile --- test/cpp/microbenchmarks/fullstack_fixtures.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cpp/microbenchmarks/fullstack_fixtures.h b/test/cpp/microbenchmarks/fullstack_fixtures.h index dc297010599..acc56bf39bf 100644 --- a/test/cpp/microbenchmarks/fullstack_fixtures.h +++ b/test/cpp/microbenchmarks/fullstack_fixtures.h @@ -212,8 +212,8 @@ class EndpointPairFixture : public BaseFixture { class SockPair : public EndpointPairFixture { public: SockPair(Service* service) - : EndpointPairFixture(service, grpc_iomgr_create_endpoint_pair( - "test", Library::get().rq(), 8192)) {} + : EndpointPairFixture(service, + grpc_iomgr_create_endpoint_pair("test", NULL)) {} }; class InProcessCHTTP2 : public EndpointPairFixture { From 846242dd5e7bce2a63b3edef96058fe371e6f508 Mon Sep 17 00:00:00 2001 From: Menghan Li Date: Fri, 24 Mar 2017 14:26:55 -0700 Subject: [PATCH 36/90] update go version in Dockerfile to 1.8 --- .../dockerfile/interoptest/grpc_interop_go/Dockerfile.template | 2 +- .../interoptest/grpc_interop_http2/Dockerfile.template | 2 +- .../stress_test/grpc_interop_stress_go/Dockerfile.template | 2 +- tools/dockerfile/interoptest/grpc_interop_go/Dockerfile | 2 +- tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile | 2 +- tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template index 38a5ca725da..bf025d80371 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile.template @@ -29,7 +29,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - FROM golang:1.5 + FROM golang:latest <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template index 6204c3e2cb2..0899e007d9e 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template @@ -29,7 +29,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - FROM golang:1.5 + FROM golang:latest <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> diff --git a/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template index e02254cd535..ad8ad71b5fa 100644 --- a/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile.template @@ -29,7 +29,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - FROM golang:1.5 + FROM golang:latest <%include file="../../gcp_api_libraries.include"/> <%include file="../../python_deps.include"/> diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index d3bf071c72e..d7a0b1786b9 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 66d9b4f6403..42e9edf98ad 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile index bbf7de7f91d..c099f339aee 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_go/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM golang:1.5 +FROM golang:latest # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean From a1eefcea5f03a8e382db9fdd3222f4e394a4fe3b Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 31 Mar 2017 11:17:28 -0700 Subject: [PATCH 37/90] Improve Node benchmarks, add generic unary test --- src/node/performance/benchmark_server.js | 8 +++++++ src/node/performance/generic_service.js | 11 ++++++++- src/node/performance/worker_service_impl.js | 23 ++++++++++++++++++- .../performance/build_performance.sh | 3 +++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/node/performance/benchmark_server.js b/src/node/performance/benchmark_server.js index 6abde2e17af..7158af775a4 100644 --- a/src/node/performance/benchmark_server.js +++ b/src/node/performance/benchmark_server.js @@ -88,6 +88,13 @@ function streamingCall(call) { }); } +function makeUnaryGenericCall(response_size) { + var response = zeroBuffer(response_size); + return function unaryGenericCall(call, callback) { + callback(null, response); + }; +} + function makeStreamingGenericCall(response_size) { var response = zeroBuffer(response_size); return function streamingGenericCall(call) { @@ -129,6 +136,7 @@ function BenchmarkServer(host, port, tls, generic, response_size) { this.port = server.bind(host + ':' + port, server_creds); if (generic) { server.addService(genericService, { + unaryCall: makeUnaryGenericCall(response_size), streamingCall: makeStreamingGenericCall(response_size) }); } else { diff --git a/src/node/performance/generic_service.js b/src/node/performance/generic_service.js index ce09cc4336c..c936ac30bca 100644 --- a/src/node/performance/generic_service.js +++ b/src/node/performance/generic_service.js @@ -34,8 +34,17 @@ var _ = require('lodash'); module.exports = { + 'unaryCall' : { + path: '/grpc.testing.BenchmarkService/UnaryCall', + requestStream: false, + responseStream: false, + requestSerialize: _.identity, + requestDeserialize: _.identity, + responseSerialize: _.identity, + responseDeserialize: _.identity + }, 'streamingCall' : { - path: '/grpc.testing/BenchmarkService', + path: '/grpc.testing.BenchmarkService/StreamingCall', requestStream: true, responseStream: true, requestSerialize: _.identity, diff --git a/src/node/performance/worker_service_impl.js b/src/node/performance/worker_service_impl.js index 38888a7219a..fa864f99255 100644 --- a/src/node/performance/worker_service_impl.js +++ b/src/node/performance/worker_service_impl.js @@ -89,6 +89,7 @@ module.exports = function WorkerServiceImpl(benchmark_impl, server) { default: call.emit('error', new Error('Unsupported PayloadConfig type' + setup.payload_config.payload)); + return; } switch (setup.load_params.load) { case 'closed_loop': @@ -103,6 +104,7 @@ module.exports = function WorkerServiceImpl(benchmark_impl, server) { default: call.emit('error', new Error('Unsupported LoadParams type' + setup.load_params.load)); + return; } stats = client.mark(); call.write({ @@ -137,8 +139,27 @@ module.exports = function WorkerServiceImpl(benchmark_impl, server) { switch (request.argtype) { case 'setup': console.log('ServerConfig %j', request.setup); + var setup = request.setup; + var resp_size, generic; + if (setup.payload_config) { + switch (setup.payload_config.payload) { + case 'bytebuf_params': + resp_size = setup.payload_config.bytebuf_params.resp_size; + generic = true; + break; + case 'simple_params': + resp_size = setup.payload_config.simple_params.resp_size; + generic = false; + break; + default: + call.emit('error', new Error('Unsupported PayloadConfig type' + + setup.payload_config.payload)); + return; + } + } server = new BenchmarkServer('[::]', request.setup.port, - request.setup.security_params); + request.setup.security_params, + generic, resp_size); server.on('started', function() { stats = server.mark(); call.write({ diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh index 5f8749dda27..3f7c9fed961 100755 --- a/tools/run_tests/performance/build_performance.sh +++ b/tools/run_tests/performance/build_performance.sh @@ -61,6 +61,9 @@ do "csharp") python tools/run_tests/run_tests.py -l $language -c $CONFIG --build_only -j 8 --compiler coreclr ;; + "node") + python tools/run_tests/run_tests.py -l $language -c $CONFIG --build_only -j 8 --iomgr_platform=uv + ;; *) python tools/run_tests/run_tests.py -l $language -c $CONFIG --build_only -j 8 ;; From c64601421b7a4072909386c2e79285fe4b487d9f Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 31 Mar 2017 11:22:52 -0700 Subject: [PATCH 38/90] Large message latency improvements: remove a memcpy and a Buffer construtor call --- src/node/ext/byte_buffer.cc | 39 +++++++++++++------------------------ src/node/ext/byte_buffer.h | 4 ---- src/node/ext/slice.cc | 5 ++--- 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/src/node/ext/byte_buffer.cc b/src/node/ext/byte_buffer.cc index 7d6fb198603..a99b96bea54 100644 --- a/src/node/ext/byte_buffer.cc +++ b/src/node/ext/byte_buffer.cc @@ -45,6 +45,7 @@ namespace grpc { namespace node { +using Nan::Callback; using Nan::MaybeLocal; using v8::Function; @@ -62,7 +63,11 @@ grpc_byte_buffer *BufferToByteBuffer(Local buffer) { } namespace { -void delete_buffer(char *data, void *hint) { delete[] data; } +void delete_buffer(char *data, void *hint) { + grpc_slice *slice = static_cast(hint); + grpc_slice_unref(*slice); + delete slice; +} } Local ByteBufferToBuffer(grpc_byte_buffer *buffer) { @@ -75,31 +80,15 @@ Local ByteBufferToBuffer(grpc_byte_buffer *buffer) { Nan::ThrowError("Error initializing byte buffer reader."); return scope.Escape(Nan::Undefined()); } - grpc_slice slice = grpc_byte_buffer_reader_readall(&reader); - size_t length = GRPC_SLICE_LENGTH(slice); - char *result = new char[length]; - memcpy(result, GRPC_SLICE_START_PTR(slice), length); - grpc_slice_unref(slice); - return scope.Escape(MakeFastBuffer( - Nan::NewBuffer(result, length, delete_buffer, NULL).ToLocalChecked())); + grpc_slice *slice = new grpc_slice; + *slice = grpc_byte_buffer_reader_readall(&reader); + grpc_byte_buffer_reader_destroy(&reader); + char *result = reinterpret_cast(GRPC_SLICE_START_PTR(*slice)); + size_t length = GRPC_SLICE_LENGTH(*slice); + Local buf = + Nan::NewBuffer(result, length, delete_buffer, slice).ToLocalChecked(); + return scope.Escape(buf); } -Local MakeFastBuffer(Local slowBuffer) { - Nan::EscapableHandleScope scope; - Local globalObj = Nan::GetCurrentContext()->Global(); - MaybeLocal constructorValue = Nan::Get( - globalObj, Nan::New("Buffer").ToLocalChecked()); - Local bufferConstructor = Local::Cast( - constructorValue.ToLocalChecked()); - const int argc = 3; - Local consArgs[argc] = { - slowBuffer, - Nan::New(::node::Buffer::Length(slowBuffer)), - Nan::New(0) - }; - MaybeLocal fastBuffer = Nan::NewInstance(bufferConstructor, - argc, consArgs); - return scope.Escape(fastBuffer.ToLocalChecked()); -} } // namespace node } // namespace grpc diff --git a/src/node/ext/byte_buffer.h b/src/node/ext/byte_buffer.h index 55bc0ab3771..e8c4ac90bdd 100644 --- a/src/node/ext/byte_buffer.h +++ b/src/node/ext/byte_buffer.h @@ -50,10 +50,6 @@ grpc_byte_buffer *BufferToByteBuffer(v8::Local buffer); /* Convert a grpc_byte_buffer to a Node.js Buffer */ v8::Local ByteBufferToBuffer(grpc_byte_buffer *buffer); -/* Convert a ::node::Buffer to a fast Buffer, as defined in the Node - Buffer documentation */ -v8::Local MakeFastBuffer(v8::Local slowBuffer); - } // namespace node } // namespace grpc diff --git a/src/node/ext/slice.cc b/src/node/ext/slice.cc index 98a80b3d2f9..104dd9e22cd 100644 --- a/src/node/ext/slice.cc +++ b/src/node/ext/slice.cc @@ -37,7 +37,6 @@ #include #include "slice.h" -#include "byte_buffer.h" namespace grpc { namespace node { @@ -93,9 +92,9 @@ Local CreateBufferFromSlice(const grpc_slice slice) { Nan::EscapableHandleScope scope; grpc_slice *slice_ptr = new grpc_slice; *slice_ptr = grpc_slice_ref(slice); - return scope.Escape(MakeFastBuffer(Nan::NewBuffer( + return scope.Escape(Nan::NewBuffer( const_cast(reinterpret_cast(GRPC_SLICE_START_PTR(*slice_ptr))), - GRPC_SLICE_LENGTH(*slice_ptr), SliceFreeCallback, slice_ptr).ToLocalChecked())); + GRPC_SLICE_LENGTH(*slice_ptr), SliceFreeCallback, slice_ptr).ToLocalChecked()); } } // namespace node From 0be726b34dafa05a46f5071d9786692d23e7a503 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 3 Apr 2017 20:04:34 +0200 Subject: [PATCH 39/90] bump version to 1.2.2 --- BUILD | 2 +- CMakeLists.txt | 2 +- Makefile | 4 ++-- build.yaml | 2 +- gRPC-Core.podspec | 2 +- gRPC-ProtoRPC.podspec | 2 +- gRPC-RxLibrary.podspec | 2 +- gRPC.podspec | 2 +- package.json | 2 +- package.xml | 4 ++-- src/cpp/common/version_cc.cc | 2 +- src/csharp/Grpc.Auth/project.json | 4 ++-- src/csharp/Grpc.Core.Testing/project.json | 4 ++-- src/csharp/Grpc.Core/VersionInfo.cs | 4 ++-- src/csharp/Grpc.Core/project.json | 2 +- src/csharp/Grpc.HealthCheck/project.json | 4 ++-- src/csharp/Grpc.Reflection/project.json | 4 ++-- src/csharp/build_packages_dotnetcli.bat | 2 +- src/csharp/build_packages_dotnetcli.sh | 4 ++-- src/node/health_check/package.json | 4 ++-- src/node/tools/package.json | 2 +- src/objective-c/!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/GRPCClient/private/version.h | 2 +- src/php/composer.json | 2 +- src/python/grpcio/grpc_version.py | 2 +- src/python/grpcio_health_checking/grpc_version.py | 2 +- src/python/grpcio_reflection/grpc_version.py | 2 +- src/python/grpcio_tests/grpc_version.py | 2 +- src/ruby/lib/grpc/version.rb | 2 +- src/ruby/tools/version.rb | 2 +- tools/distrib/python/grpcio_tools/grpc_version.py | 2 +- tools/doxygen/Doxyfile.c++ | 2 +- tools/doxygen/Doxyfile.c++.internal | 2 +- 33 files changed, 42 insertions(+), 42 deletions(-) diff --git a/BUILD b/BUILD index 0c6daee4bb8..0e8b058cb5f 100644 --- a/BUILD +++ b/BUILD @@ -41,7 +41,7 @@ g_stands_for = "green" core_version = "3.0.0-dev" -version = "1.2.1" +version = "1.2.2" grpc_cc_library( name = "gpr", diff --git a/CMakeLists.txt b/CMakeLists.txt index f9ca86266bd..4496d37bd6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.2.1") +set(PACKAGE_VERSION "1.2.2") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") diff --git a/Makefile b/Makefile index 5655ae36858..00a99ab964f 100644 --- a/Makefile +++ b/Makefile @@ -412,8 +412,8 @@ Q = @ endif CORE_VERSION = 3.0.0-dev -CPP_VERSION = 1.2.1 -CSHARP_VERSION = 1.2.1 +CPP_VERSION = 1.2.2 +CSHARP_VERSION = 1.2.2 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) diff --git a/build.yaml b/build.yaml index 1c3a23bf6b7..8df00f2c4bd 100644 --- a/build.yaml +++ b/build.yaml @@ -14,7 +14,7 @@ settings: '#10': See the expand_version.py for all the quirks here core_version: 3.0.0-dev g_stands_for: green - version: 1.2.1 + version: 1.2.2 filegroups: - name: census public_headers: diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 585713c0e61..51e8fdfa02e 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -37,7 +37,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.2.1' + version = '1.2.2' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index cc9817a02a9..86c51819227 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.2.1' + version = '1.2.2' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'http://www.grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index e631ee5104a..ba365ceaa04 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.2.1' + version = '1.2.2' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'http://www.grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index ea78d5f91ea..b51c4df5dae 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -35,7 +35,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.2.1' + version = '1.2.2' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'http://www.grpc.io' diff --git a/package.json b/package.json index 31846bf2e79..b4503d00dba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.2.1", + "version": "1.2.2", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "http://www.grpc.io/", diff --git a/package.xml b/package.xml index 24a0bf01a8b..6ff9a9c9cc4 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ 2017-03-01 - 1.2.1 - 1.2.1 + 1.2.2 + 1.2.2 beta diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index 5cac8cdf781..6405fbcbd7d 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -37,5 +37,5 @@ #include namespace grpc { -grpc::string Version() { return "1.2.1"; } +grpc::string Version() { return "1.2.2"; } } diff --git a/src/csharp/Grpc.Auth/project.json b/src/csharp/Grpc.Auth/project.json index 66b3fa93d6a..31022e65bdf 100644 --- a/src/csharp/Grpc.Auth/project.json +++ b/src/csharp/Grpc.Auth/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1", + "version": "1.2.2", "title": "gRPC C# Auth", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1", + "Grpc.Core": "1.2.2", "Google.Apis.Auth": "1.21.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Core.Testing/project.json b/src/csharp/Grpc.Core.Testing/project.json index c4e015f4186..22d728fc159 100644 --- a/src/csharp/Grpc.Core.Testing/project.json +++ b/src/csharp/Grpc.Core.Testing/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1", + "version": "1.2.2", "title": "gRPC C# Core Testing", "authors": [ "Google Inc." ], "copyright": "Copyright 2017, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1" + "Grpc.Core": "1.2.2" }, "frameworks": { "net45": { diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 42881c9f065..d46f863e173 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -48,11 +48,11 @@ namespace Grpc.Core /// /// Current AssemblyFileVersion of gRPC C# assemblies /// - public const string CurrentAssemblyFileVersion = "1.2.1.0"; + public const string CurrentAssemblyFileVersion = "1.2.2.0"; /// /// Current version of gRPC C# /// - public const string CurrentVersion = "1.2.1"; + public const string CurrentVersion = "1.2.2"; } } diff --git a/src/csharp/Grpc.Core/project.json b/src/csharp/Grpc.Core/project.json index f9f617772a8..0d56b55fe30 100644 --- a/src/csharp/Grpc.Core/project.json +++ b/src/csharp/Grpc.Core/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1", + "version": "1.2.2", "title": "gRPC C# Core", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", diff --git a/src/csharp/Grpc.HealthCheck/project.json b/src/csharp/Grpc.HealthCheck/project.json index 0a564fde5af..259d973ff62 100644 --- a/src/csharp/Grpc.HealthCheck/project.json +++ b/src/csharp/Grpc.HealthCheck/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1", + "version": "1.2.2", "title": "gRPC C# Healthchecking", "authors": [ "Google Inc." ], "copyright": "Copyright 2015, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1", + "Grpc.Core": "1.2.2", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/Grpc.Reflection/project.json b/src/csharp/Grpc.Reflection/project.json index ac948d45f04..b40945cad40 100644 --- a/src/csharp/Grpc.Reflection/project.json +++ b/src/csharp/Grpc.Reflection/project.json @@ -1,5 +1,5 @@ { - "version": "1.2.1", + "version": "1.2.2", "title": "gRPC C# Reflection", "authors": [ "Google Inc." ], "copyright": "Copyright 2016, Google Inc.", @@ -21,7 +21,7 @@ } }, "dependencies": { - "Grpc.Core": "1.2.1", + "Grpc.Core": "1.2.2", "Google.Protobuf": "3.2.0" }, "frameworks": { diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index da6556dc27e..51c87246bf0 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -28,7 +28,7 @@ @rem OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. @rem Current package versions -set VERSION=1.2.1 +set VERSION=1.2.2 set PROTOBUF_VERSION=3.0.0 @rem Adjust the location of nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index c0d58d667a9..7e7a029fcfd 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -66,7 +66,7 @@ dotnet pack --configuration Release Grpc.Auth/project.json --output ../../artifa dotnet pack --configuration Release Grpc.HealthCheck/project.json --output ../../artifacts dotnet pack --configuration Release Grpc.Reflection/project.json --output ../../artifacts -nuget pack Grpc.nuspec -Version "1.2.1" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.2.1" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.2.2" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.2.2" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/node/health_check/package.json b/src/node/health_check/package.json index 177b27b9112..61bb83f229a 100644 --- a/src/node/health_check/package.json +++ b/src/node/health_check/package.json @@ -1,6 +1,6 @@ { "name": "grpc-health-check", - "version": "1.2.1", + "version": "1.2.2", "author": "Google Inc.", "description": "Health check service for use with gRPC", "repository": { @@ -15,7 +15,7 @@ } ], "dependencies": { - "grpc": "^1.2.1", + "grpc": "^1.2.2", "lodash": "^3.9.3", "google-protobuf": "^3.0.0" }, diff --git a/src/node/tools/package.json b/src/node/tools/package.json index 5867db6bb9b..460f2fc857a 100644 --- a/src/node/tools/package.json +++ b/src/node/tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.2.1", + "version": "1.2.2", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "http://www.grpc.io/", diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 226e36fd866..ca5ad7b2f56 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.2.1' + v = '1.2.2' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 60866a078af..d04f73d3188 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -38,4 +38,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.2.1" +#define GRPC_OBJC_VERSION_STRING @"1.2.2" diff --git a/src/php/composer.json b/src/php/composer.json index ee7616b45df..fa4e3e5990f 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "BSD-3-Clause", - "version": "1.2.1", + "version": "1.2.2", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.1.0" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 330c20dd8f3..caf940273e2 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION='1.2.1' +VERSION='1.2.2' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index c06086d5b8c..7b7d95daacf 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION='1.2.1' +VERSION='1.2.2' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index f774e403f50..dffabad634f 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION='1.2.1' +VERSION='1.2.2' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 9a1a472afcc..d31ddd3c221 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION='1.2.1' +VERSION='1.2.2' diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 3ee48ae328f..cc25ca10a0e 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -29,5 +29,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.2.1' + VERSION = '1.2.2' end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index d037014852f..7c7eada243d 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -29,6 +29,6 @@ module GRPC module Tools - VERSION = '1.2.1' + VERSION = '1.2.2' end end diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index e125d189220..a28f2448d4d 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -29,4 +29,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION='1.2.1' +VERSION='1.2.2' diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 4d427640503..70c7f6d5ae8 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.1 +PROJECT_NUMBER = 1.2.2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 8e5730a0d32..34d6a59be20 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.2.1 +PROJECT_NUMBER = 1.2.2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From a5ea8f2150825e14aa9da1758fd02cff37035ebb Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 3 Apr 2017 16:03:58 -0700 Subject: [PATCH 40/90] Fix serializer error handling, update ProtoBuf.js dependency --- package.json | 2 +- src/node/src/protobuf_js_6_common.js | 2 +- templates/package.json.template | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 440db2ffc8f..7f242326d7e 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "lodash": "^4.15.0", "nan": "^2.0.0", "node-pre-gyp": "^0.6.0", - "protobufjs": "^6.0.0", + "protobufjs": "^6.7.0", "cares": "^1.1.5" }, "devDependencies": { diff --git a/src/node/src/protobuf_js_6_common.js b/src/node/src/protobuf_js_6_common.js index baa62cce866..7e523731d3b 100644 --- a/src/node/src/protobuf_js_6_common.js +++ b/src/node/src/protobuf_js_6_common.js @@ -80,7 +80,7 @@ exports.serializeCls = function serializeCls(cls) { var message = cls.fromObject(arg); var errMsg = cls.verify(message); if (errMsg) { - throw errMsg; + throw Error(errMsg); } return cls.encode(message).finish(); }; diff --git a/templates/package.json.template b/templates/package.json.template index 11fb0fb9206..b69fd28d2ab 100644 --- a/templates/package.json.template +++ b/templates/package.json.template @@ -36,7 +36,7 @@ "lodash": "^4.15.0", "nan": "^2.0.0", "node-pre-gyp": "^0.6.0", - "protobufjs": "^6.0.0", + "protobufjs": "^6.7.0", "cares": "^1.1.5" }, "devDependencies": { From c84d67e85b7939abffab996995562be06edb0854 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 3 Apr 2017 16:52:03 -0700 Subject: [PATCH 41/90] Keep changes from #10093 --- src/node/src/protobuf_js_5_common.js | 1 + src/node/src/protobuf_js_6_common.js | 1 + 2 files changed, 2 insertions(+) diff --git a/src/node/src/protobuf_js_5_common.js b/src/node/src/protobuf_js_5_common.js index b16228b9ee8..62cf2f4acaa 100644 --- a/src/node/src/protobuf_js_5_common.js +++ b/src/node/src/protobuf_js_5_common.js @@ -120,6 +120,7 @@ exports.getProtobufServiceAttrs = function getProtobufServiceAttrs(service, return _.camelCase(method.name); }), _.map(service.children, function(method) { return { + originalName: method.name, path: prefix + method.name, requestStream: method.requestStream, responseStream: method.responseStream, diff --git a/src/node/src/protobuf_js_6_common.js b/src/node/src/protobuf_js_6_common.js index 7e523731d3b..cac0f711451 100644 --- a/src/node/src/protobuf_js_6_common.js +++ b/src/node/src/protobuf_js_6_common.js @@ -121,6 +121,7 @@ exports.getProtobufServiceAttrs = function getProtobufServiceAttrs(service, return _.camelCase(method.name); }), _.map(service.methods, function(method) { return { + originalName: method.name, path: prefix + method.name, requestStream: !!method.requestStream, responseStream: !!method.responseStream, From 5272dfca336ca9f3593e4a3070d449432ee65621 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 27 Feb 2017 20:39:35 -0800 Subject: [PATCH 42/90] Reset the keepalive timer when sending a bdp ping --- .../chttp2/transport/chttp2_transport.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 26f9449f4b9..01f74247dcb 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -488,7 +488,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->ping_policy.max_pings_without_data; t->ping_state.is_delayed_ping_timer_set = false; - /** Start client-side keepalive pings */ + /* Start client-side keepalive pings */ if (t->is_client) { t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_WAITING; GRPC_CHTTP2_REF_TRANSPORT(t, "init keepalive ping"); @@ -2114,6 +2114,10 @@ static void start_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, if (grpc_http_trace) { gpr_log(GPR_DEBUG, "%s: Start BDP ping", t->peer_string); } + /* Reset the keepalive ping timer */ + if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING) { + grpc_timer_cancel(exec_ctx, &t->keepalive_ping_timer); + } grpc_bdp_estimator_start_ping(&t->bdp_estimator); } @@ -2172,6 +2176,13 @@ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), t->keepalive_time), &t->init_keepalive_ping_locked, gpr_now(GPR_CLOCK_MONOTONIC)); } + } else if (error == GRPC_ERROR_CANCELLED && !(t->destroying || t->closed)) { + /* The keepalive ping timer may be cancelled by bdp */ + GRPC_CHTTP2_REF_TRANSPORT(t, "init keepalive ping"); + grpc_timer_init( + exec_ctx, &t->keepalive_ping_timer, + gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC), t->keepalive_time), + &t->init_keepalive_ping_locked, gpr_now(GPR_CLOCK_MONOTONIC)); } GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "init keepalive ping"); } @@ -2213,8 +2224,8 @@ static void keepalive_watchdog_fired_locked(grpc_exec_ctx *exec_ctx, void *arg, "keepalive watchdog timeout")); } } else { - /** The watchdog timer should have been cancelled by - finish_keepalive_ping_locked. */ + /* The watchdog timer should have been cancelled by + * finish_keepalive_ping_locked. */ if (error != GRPC_ERROR_CANCELLED) { gpr_log(GPR_ERROR, "keepalive_ping_end state error: %d (expect: %d)", t->keepalive_state, GRPC_CHTTP2_KEEPALIVE_STATE_PINGING); From 0937fc1a30004bda3614fb8507cc15d775ab9df2 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 27 Feb 2017 23:19:00 -0800 Subject: [PATCH 43/90] Client-side keepalive pings server enforcement --- include/grpc/impl/codegen/grpc_types.h | 5 ++++ .../chttp2/transport/chttp2_transport.c | 26 ++++++++++++++++++- .../transport/chttp2/transport/frame_ping.c | 17 ++++++++++++ .../ext/transport/chttp2/transport/internal.h | 9 +++++++ .../ext/transport/chttp2/transport/writing.c | 20 ++++++++++++++ 5 files changed, 76 insertions(+), 1 deletion(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index aa4210b1a76..a84ce6fa2bf 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -202,6 +202,11 @@ typedef struct { a data frame or header frame) */ #define GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA \ "grpc.http2.max_pings_without_data" +/** How many misbehaving pings the server can bear before sending goaway and + closing the transport? + (0 indicates that the server can bear an infinite number of misbehaving + pings) */ +#define GRPC_ARG_HTTP2_MAX_PING_STRIKES "grpc.http2.max_ping_strikes" /** How much data are we willing to queue up per stream if GRPC_WRITE_BUFFER_HINT is set? This is an upper bound */ #define GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE "grpc.http2.write_buffer_size" diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 01f74247dcb..195b34a3773 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -153,6 +153,7 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, #define DEFAULT_MIN_TIME_BETWEEN_PINGS_MS 0 #define DEFAULT_MAX_PINGS_BETWEEN_DATA 3 +#define DEFAULT_MAX_PING_STRIKES 2 /** keepalive-relevant functions */ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, @@ -351,6 +352,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, .max_pings_without_data = DEFAULT_MAX_PINGS_BETWEEN_DATA, .min_time_between_pings = gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN), + .max_ping_strikes = DEFAULT_MAX_PING_STRIKES, }; /* client-side keepalive setting */ @@ -396,6 +398,11 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->ping_policy.max_pings_without_data = grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){DEFAULT_MAX_PINGS_BETWEEN_DATA, 0, INT_MAX}); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_MAX_PING_STRIKES)) { + t->ping_policy.max_ping_strikes = grpc_channel_arg_get_integer( + &channel_args->args[i], + (grpc_integer_options){DEFAULT_MAX_PINGS_BETWEEN_DATA, 0, INT_MAX}); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS)) { t->ping_policy.min_time_between_pings = gpr_time_from_millis( @@ -488,6 +495,9 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->ping_policy.max_pings_without_data; t->ping_state.is_delayed_ping_timer_set = false; + t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); + t->ping_recv_state.ping_strikes = 0; + /* Start client-side keepalive pings */ if (t->is_client) { t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_WAITING; @@ -1448,6 +1458,19 @@ static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ERROR_UNREF(error); } +void grpc_chttp2_ping_strike(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { + if (++t->ping_recv_state.ping_strikes > t->ping_policy.max_ping_strikes && + t->ping_policy.max_ping_strikes != 0) { + send_goaway(exec_ctx, t, + grpc_error_set_int(GRPC_ERROR_CREATE("too_many_pings"), + GRPC_ERROR_INT_HTTP2_ERROR, + GRPC_HTTP2_ENHANCE_YOUR_CALM)); + /*The transport will be closed after the write is done */ + close_transport_locked(exec_ctx, t, GRPC_ERROR_CREATE("Too many pings")); + } +} + static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, grpc_error *error_ignored) { @@ -2163,7 +2186,8 @@ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_chttp2_transport *t = arg; GPR_ASSERT(t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING); if (error == GRPC_ERROR_NONE && !(t->destroying || t->closed)) { - if (t->keepalive_permit_without_calls || t->stream_map.count > 0) { + if (t->keepalive_permit_without_calls || + grpc_chttp2_stream_map_size(&t->stream_map) > 0) { t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_PINGING; GRPC_CHTTP2_REF_TRANSPORT(t, "keepalive ping end"); send_ping_locked(exec_ctx, t, GRPC_CHTTP2_PING_ON_NEXT_WRITE, diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 46dafdb62f6..1398e55953f 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -103,6 +103,23 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, if (p->is_ack) { grpc_chttp2_ack_ping(exec_ctx, t, p->opaque_8bytes); } else { + if (!t->is_client) { + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec elapsed = + gpr_time_sub(now, t->ping_recv_state.last_ping_recv_time); + if (t->keepalive_permit_without_calls == 0 && + grpc_chttp2_stream_map_size(&t->stream_map) == 0) { + if (gpr_time_cmp(elapsed, gpr_time_from_seconds(7200, GPR_TIMESPAN)) < + 0) { + grpc_chttp2_ping_strike(exec_ctx, t); + } + } else { + if (gpr_time_cmp(elapsed, t->ping_policy.min_time_between_pings) < + 0) { + grpc_chttp2_ping_strike(exec_ctx, t); + } + } + } if (!g_disable_ping_ack) { if (t->ping_ack_count == t->ping_ack_capacity) { t->ping_ack_capacity = GPR_MAX(t->ping_ack_capacity * 3 / 2, 3); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 50993e4ecdf..889d8ce7547 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -97,6 +97,7 @@ typedef struct { typedef struct { gpr_timespec min_time_between_pings; int max_pings_without_data; + int max_ping_strikes; } grpc_chttp2_repeated_ping_policy; typedef struct { @@ -106,6 +107,11 @@ typedef struct { bool is_delayed_ping_timer_set; } grpc_chttp2_repeated_ping_state; +typedef struct { + gpr_timespec last_ping_recv_time; + int ping_strikes; +} grpc_chttp2_server_ping_recv_state; + /* deframer state for the overall http2 stream of bytes */ typedef enum { /* prefix: one entry per http2 connection prefix byte */ @@ -316,6 +322,7 @@ struct grpc_chttp2_transport { size_t ping_ack_count; size_t ping_ack_capacity; uint64_t *ping_acks; + grpc_chttp2_server_ping_recv_state ping_recv_state; /** parser for headers */ grpc_chttp2_hpack_parser hpack_parser; @@ -792,6 +799,8 @@ void grpc_chttp2_incoming_byte_stream_finished( void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, uint64_t id); +void grpc_chttp2_ping_strike(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t); + typedef enum { /* don't initiate a transport write, but piggyback on the next one */ GRPC_CHTTP2_STREAM_WRITE_PIGGYBACK, diff --git a/src/core/ext/transport/chttp2/transport/writing.c b/src/core/ext/transport/chttp2/transport/writing.c index 0869056f56d..be41b3d1865 100644 --- a/src/core/ext/transport/chttp2/transport/writing.c +++ b/src/core/ext/transport/chttp2/transport/writing.c @@ -229,6 +229,11 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, now_writing = true; t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; + if (!t->is_client) { + t->ping_recv_state.last_ping_recv_time = + gpr_inf_past(GPR_CLOCK_MONOTONIC); + t->ping_recv_state.ping_strikes = 0; + } } /* send any window updates */ if (s->announce_window > 0) { @@ -238,6 +243,11 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, s->id, s->announce_window, &s->stats.outgoing)); t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; + if (!t->is_client) { + t->ping_recv_state.last_ping_recv_time = + gpr_inf_past(GPR_CLOCK_MONOTONIC); + t->ping_recv_state.ping_strikes = 0; + } GRPC_CHTTP2_FLOW_DEBIT_STREAM("write", t, s, announce_window, announce); } if (sent_initial_metadata) { @@ -270,6 +280,11 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, send_bytes); t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; + if (!t->is_client) { + t->ping_recv_state.last_ping_recv_time = + gpr_inf_past(GPR_CLOCK_MONOTONIC); + t->ping_recv_state.ping_strikes = 0; + } if (is_last_frame) { s->send_trailing_metadata = NULL; s->sent_trailing_metadata = true; @@ -345,6 +360,11 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx, 0, announced, &throwaway_stats)); t->ping_state.pings_before_data_required = t->ping_policy.max_pings_without_data; + if (!t->is_client) { + t->ping_recv_state.last_ping_recv_time = + gpr_inf_past(GPR_CLOCK_MONOTONIC); + t->ping_recv_state.ping_strikes = 0; + } } for (size_t i = 0; i < t->ping_ack_count; i++) { From 187c752c6f645daaf42fa989deca601c4685de58 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 28 Feb 2017 14:07:13 -0800 Subject: [PATCH 44/90] Add min_ping_interval_without_data --- include/grpc/impl/codegen/grpc_types.h | 4 +++ .../chttp2/transport/chttp2_transport.c | 13 +++++++++- .../transport/chttp2/transport/frame_ping.c | 25 +++++++++++-------- .../ext/transport/chttp2/transport/internal.h | 1 + 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index a84ce6fa2bf..cb22785b354 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -207,6 +207,10 @@ typedef struct { (0 indicates that the server can bear an infinite number of misbehaving pings) */ #define GRPC_ARG_HTTP2_MAX_PING_STRIKES "grpc.http2.max_ping_strikes" +/** Minimum allowed time between two pings without sending any data frame. Int + valued, seconds */ +#define GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_S \ + "grpc.http2.min_ping_interval_without_data" /** How much data are we willing to queue up per stream if GRPC_WRITE_BUFFER_HINT is set? This is an upper bound */ #define GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE "grpc.http2.write_buffer_size" diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 195b34a3773..c4deb24e41e 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -154,6 +154,7 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, #define DEFAULT_MIN_TIME_BETWEEN_PINGS_MS 0 #define DEFAULT_MAX_PINGS_BETWEEN_DATA 3 #define DEFAULT_MAX_PING_STRIKES 2 +#define DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_S 300 /** keepalive-relevant functions */ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, @@ -353,6 +354,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, .min_time_between_pings = gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN), .max_ping_strikes = DEFAULT_MAX_PING_STRIKES, + .min_ping_interval_without_data = gpr_time_from_seconds( + DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_S, GPR_TIMESPAN), }; /* client-side keepalive setting */ @@ -402,7 +405,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ARG_HTTP2_MAX_PING_STRIKES)) { t->ping_policy.max_ping_strikes = grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){DEFAULT_MAX_PINGS_BETWEEN_DATA, 0, INT_MAX}); + (grpc_integer_options){DEFAULT_MAX_PING_STRIKES, 0, INT_MAX}); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS)) { t->ping_policy.min_time_between_pings = gpr_time_from_millis( @@ -411,6 +414,14 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, (grpc_integer_options){DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, 0, INT_MAX}), GPR_TIMESPAN); + } else if (0 == strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_S)) { + t->ping_policy.min_ping_interval_without_data = gpr_time_from_seconds( + grpc_channel_arg_get_integer( + &channel_args->args[i], + (grpc_integer_options){DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_S, + 0, INT_MAX}), + GPR_TIMESPAN); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { t->write_buffer_size = (uint32_t)grpc_channel_arg_get_integer( diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 1398e55953f..98f53bf3619 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -105,19 +105,22 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, } else { if (!t->is_client) { gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_timespec elapsed = - gpr_time_sub(now, t->ping_recv_state.last_ping_recv_time); + gpr_timespec next_allowed_ping = + gpr_time_add(t->ping_recv_state.last_ping_recv_time, + t->ping_policy.min_ping_interval_without_data); + if (t->keepalive_permit_without_calls == 0 && grpc_chttp2_stream_map_size(&t->stream_map) == 0) { - if (gpr_time_cmp(elapsed, gpr_time_from_seconds(7200, GPR_TIMESPAN)) < - 0) { - grpc_chttp2_ping_strike(exec_ctx, t); - } - } else { - if (gpr_time_cmp(elapsed, t->ping_policy.min_time_between_pings) < - 0) { - grpc_chttp2_ping_strike(exec_ctx, t); - } + /* The “2 hours” restricts the number of PINGS to an implementation + equivalent to TCP Keep-Alive, whose interval is specified to + default to no less than two hours in RFC1122. */ + next_allowed_ping = + gpr_time_add(t->ping_recv_state.last_ping_recv_time, + gpr_time_from_seconds(7200, GPR_TIMESPAN)); + } + + if (gpr_time_cmp(next_allowed_ping, now) > 0) { + grpc_chttp2_ping_strike(exec_ctx, t); } } if (!g_disable_ping_ack) { diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 889d8ce7547..c302d4ac348 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -98,6 +98,7 @@ typedef struct { gpr_timespec min_time_between_pings; int max_pings_without_data; int max_ping_strikes; + gpr_timespec min_ping_interval_without_data; } grpc_chttp2_repeated_ping_policy; typedef struct { From af5c505fc196e09233f8de70e9e850807aac003e Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 28 Feb 2017 16:24:00 -0800 Subject: [PATCH 45/90] Add bad_ping test, fix ping test --- CMakeLists.txt | 2 + Makefile | 2 + .../transport/chttp2/transport/frame_ping.c | 2 + test/core/end2end/end2end_nosec_tests.c | 8 + test/core/end2end/end2end_tests.c | 8 + test/core/end2end/gen_build_yaml.py | 1 + test/core/end2end/generate_tests.bzl | 1 + test/core/end2end/tests/bad_ping.c | 234 +++++++++ test/core/end2end/tests/ping.c | 26 +- .../generated/sources_and_headers.json | 2 + tools/run_tests/generated/tests.json | 454 ++++++++++++++++++ .../end2end_nosec_tests.vcxproj | 2 + .../end2end_nosec_tests.vcxproj.filters | 3 + .../tests/end2end_tests/end2end_tests.vcxproj | 2 + .../end2end_tests.vcxproj.filters | 3 + 15 files changed, 742 insertions(+), 8 deletions(-) create mode 100644 test/core/end2end/tests/bad_ping.c diff --git a/CMakeLists.txt b/CMakeLists.txt index e3b2558ad29..df9cb08aa9f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4391,6 +4391,7 @@ add_library(end2end_tests test/core/end2end/end2end_test_utils.c test/core/end2end/tests/authority_not_supported.c test/core/end2end/tests/bad_hostname.c + test/core/end2end/tests/bad_ping.c test/core/end2end/tests/binary_metadata.c test/core/end2end/tests/call_creds.c test/core/end2end/tests/cancel_after_accept.c @@ -4488,6 +4489,7 @@ add_library(end2end_nosec_tests test/core/end2end/end2end_test_utils.c test/core/end2end/tests/authority_not_supported.c test/core/end2end/tests/bad_hostname.c + test/core/end2end/tests/bad_ping.c test/core/end2end/tests/binary_metadata.c test/core/end2end/tests/cancel_after_accept.c test/core/end2end/tests/cancel_after_client_done.c diff --git a/Makefile b/Makefile index 7499dbbf50d..957d00dc09b 100644 --- a/Makefile +++ b/Makefile @@ -8226,6 +8226,7 @@ LIBEND2END_TESTS_SRC = \ test/core/end2end/end2end_test_utils.c \ test/core/end2end/tests/authority_not_supported.c \ test/core/end2end/tests/bad_hostname.c \ + test/core/end2end/tests/bad_ping.c \ test/core/end2end/tests/binary_metadata.c \ test/core/end2end/tests/call_creds.c \ test/core/end2end/tests/cancel_after_accept.c \ @@ -8318,6 +8319,7 @@ LIBEND2END_NOSEC_TESTS_SRC = \ test/core/end2end/end2end_test_utils.c \ test/core/end2end/tests/authority_not_supported.c \ test/core/end2end/tests/bad_hostname.c \ + test/core/end2end/tests/bad_ping.c \ test/core/end2end/tests/binary_metadata.c \ test/core/end2end/tests/cancel_after_accept.c \ test/core/end2end/tests/cancel_after_client_done.c \ diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 98f53bf3619..61082b97441 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -122,6 +122,8 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, if (gpr_time_cmp(next_allowed_ping, now) > 0) { grpc_chttp2_ping_strike(exec_ctx, t); } + + t->ping_recv_state.last_ping_recv_time = now; } if (!g_disable_ping_ack) { if (t->ping_ack_count == t->ping_ack_capacity) { diff --git a/test/core/end2end/end2end_nosec_tests.c b/test/core/end2end/end2end_nosec_tests.c index 64bdceb2111..1187e59e6cc 100644 --- a/test/core/end2end/end2end_nosec_tests.c +++ b/test/core/end2end/end2end_nosec_tests.c @@ -49,6 +49,8 @@ extern void authority_not_supported(grpc_end2end_test_config config); extern void authority_not_supported_pre_init(void); extern void bad_hostname(grpc_end2end_test_config config); extern void bad_hostname_pre_init(void); +extern void bad_ping(grpc_end2end_test_config config); +extern void bad_ping_pre_init(void); extern void binary_metadata(grpc_end2end_test_config config); extern void binary_metadata_pre_init(void); extern void cancel_after_accept(grpc_end2end_test_config config); @@ -154,6 +156,7 @@ void grpc_end2end_tests_pre_init(void) { grpc_summon_debugger_macros(); authority_not_supported_pre_init(); bad_hostname_pre_init(); + bad_ping_pre_init(); binary_metadata_pre_init(); cancel_after_accept_pre_init(); cancel_after_client_done_pre_init(); @@ -214,6 +217,7 @@ void grpc_end2end_tests(int argc, char **argv, if (argc <= 1) { authority_not_supported(config); bad_hostname(config); + bad_ping(config); binary_metadata(config); cancel_after_accept(config); cancel_after_client_done(config); @@ -275,6 +279,10 @@ void grpc_end2end_tests(int argc, char **argv, bad_hostname(config); continue; } + if (0 == strcmp("bad_ping", argv[i])) { + bad_ping(config); + continue; + } if (0 == strcmp("binary_metadata", argv[i])) { binary_metadata(config); continue; diff --git a/test/core/end2end/end2end_tests.c b/test/core/end2end/end2end_tests.c index 37c1be41337..966031af657 100644 --- a/test/core/end2end/end2end_tests.c +++ b/test/core/end2end/end2end_tests.c @@ -49,6 +49,8 @@ extern void authority_not_supported(grpc_end2end_test_config config); extern void authority_not_supported_pre_init(void); extern void bad_hostname(grpc_end2end_test_config config); extern void bad_hostname_pre_init(void); +extern void bad_ping(grpc_end2end_test_config config); +extern void bad_ping_pre_init(void); extern void binary_metadata(grpc_end2end_test_config config); extern void binary_metadata_pre_init(void); extern void call_creds(grpc_end2end_test_config config); @@ -156,6 +158,7 @@ void grpc_end2end_tests_pre_init(void) { grpc_summon_debugger_macros(); authority_not_supported_pre_init(); bad_hostname_pre_init(); + bad_ping_pre_init(); binary_metadata_pre_init(); call_creds_pre_init(); cancel_after_accept_pre_init(); @@ -217,6 +220,7 @@ void grpc_end2end_tests(int argc, char **argv, if (argc <= 1) { authority_not_supported(config); bad_hostname(config); + bad_ping(config); binary_metadata(config); call_creds(config); cancel_after_accept(config); @@ -279,6 +283,10 @@ void grpc_end2end_tests(int argc, char **argv, bad_hostname(config); continue; } + if (0 == strcmp("bad_ping", argv[i])) { + bad_ping(config); + continue; + } if (0 == strcmp("binary_metadata", argv[i])) { binary_metadata(config); continue; diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index 3c5068ff3e3..d1e510d6362 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -93,6 +93,7 @@ LOWCPU = 0.1 END2END_TESTS = { 'authority_not_supported': default_test_options, 'bad_hostname': default_test_options, + 'bad_ping': connectivity_test_options._replace(proxyable=False), 'binary_metadata': default_test_options, 'resource_quota_server': default_test_options._replace(large_writes=True, proxyable=False), diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index 1041219f039..dc0925dc9c0 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -85,6 +85,7 @@ def test_options(needs_fullstack=False, needs_dns=False, proxyable=True, # maps test names to options END2END_TESTS = { 'bad_hostname': test_options(), + 'bad_ping': test_options(), 'binary_metadata': test_options(), 'resource_quota_server': test_options(proxyable=False), 'call_creds': test_options(secure=True), diff --git a/test/core/end2end/tests/bad_ping.c b/test/core/end2end/tests/bad_ping.c new file mode 100644 index 00000000000..97b02c90722 --- /dev/null +++ b/test/core/end2end/tests/bad_ping.c @@ -0,0 +1,234 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/core/end2end/end2end_tests.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "test/core/end2end/cq_verifier.h" + +#define MAX_PING_STRIKES 1 + +static void *tag(intptr_t t) { return (void *)t; } + +static void drain_cq(grpc_completion_queue *cq) { + grpc_event ev; + do { + ev = grpc_completion_queue_next(cq, grpc_timeout_seconds_to_deadline(5), + NULL); + } while (ev.type != GRPC_QUEUE_SHUTDOWN); +} + +static void shutdown_server(grpc_end2end_test_fixture *f) { + if (!f->server) return; + grpc_server_destroy(f->server); + f->server = NULL; +} + +static void shutdown_client(grpc_end2end_test_fixture *f) { + if (!f->client) return; + grpc_channel_destroy(f->client); + f->client = NULL; +} + +static void end_test(grpc_end2end_test_fixture *f) { + shutdown_server(f); + shutdown_client(f); + + grpc_completion_queue_shutdown(f->cq); + drain_cq(f->cq); + grpc_completion_queue_destroy(f->cq); +} + +static void test_bad_ping(grpc_end2end_test_config config) { + grpc_end2end_test_fixture f = config.create_fixture(NULL, NULL); + cq_verifier *cqv = cq_verifier_create(f.cq); + grpc_arg client_a[] = {{.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS, + .value.integer = 0}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, + .value.integer = 20}}; + grpc_arg server_a[] = { + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_S, + .value.integer = 300}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MAX_PING_STRIKES, + .value.integer = MAX_PING_STRIKES}}; + grpc_channel_args client_args = {.num_args = GPR_ARRAY_SIZE(client_a), + .args = client_a}; + grpc_channel_args server_args = {.num_args = GPR_ARRAY_SIZE(server_a), + .args = server_a}; + + config.init_client(&f, &client_args); + config.init_server(&f, &server_args); + + grpc_call *c; + grpc_call *s; + gpr_timespec deadline = grpc_timeout_seconds_to_deadline(10); + grpc_op ops[6]; + grpc_op *op; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + grpc_slice details; + int was_cancelled = 2; + + c = grpc_channel_create_call( + f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, + grpc_slice_from_static_string("/foo"), + get_host_override_slice("foo.test.google.fr:1234", config), deadline, + NULL); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->data.send_initial_metadata.metadata = NULL; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->flags = 0; + op->reserved = NULL; + op++; + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + error = + grpc_server_request_call(f.server, &s, &call_details, + &request_metadata_recv, f.cq, f.cq, tag(101)); + GPR_ASSERT(GRPC_CALL_OK == error); + CQ_EXPECT_COMPLETION(cqv, tag(101), 1); + cq_verify(cqv); + + // Send too many pings to the server to trigger the punishment: + // The first ping is sent after data frames, it won't trigger a ping strike. + // Each of the following pings will trigger a ping strike, and we need at + // least (MAX_PING_STRIKES + 1) strikes to trigger the punishment. So + // (MAX_PING_STRIKES + 2) pings are needed here. + int i; + for (i = 200; i < 202 + MAX_PING_STRIKES; i++) { + grpc_channel_ping(f.client, f.cq, tag(i), NULL); + CQ_EXPECT_COMPLETION(cqv, tag(i), 1); + cq_verify(cqv); + } + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; + op->data.send_status_from_server.trailing_metadata_count = 0; + op->data.send_status_from_server.status = GRPC_STATUS_UNIMPLEMENTED; + grpc_slice status_details = grpc_slice_from_static_string("xyz"); + op->data.send_status_from_server.status_details = &status_details; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; + op->data.recv_close_on_server.cancelled = &was_cancelled; + op->flags = 0; + op->reserved = NULL; + op++; + error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(102), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(102), 1); + CQ_EXPECT_COMPLETION(cqv, tag(1), 1); + cq_verify(cqv); + + grpc_server_shutdown_and_notify(f.server, f.cq, tag(0xdead)); + CQ_EXPECT_COMPLETION(cqv, tag(0xdead), 1); + cq_verify(cqv); + + grpc_call_destroy(s); + + // The connection should be closed immediately after the misbehaved pings, + // the in-progress RPC should fail. + GPR_ASSERT(status == GRPC_STATUS_UNAVAILABLE); + GPR_ASSERT(0 == grpc_slice_str_cmp(details, "Endpoint read failed")); + GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo")); + validate_host_override_string("foo.test.google.fr:1234", call_details.host, + config); + GPR_ASSERT(was_cancelled == 1); + + grpc_slice_unref(details); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + grpc_call_destroy(c); + cq_verifier_destroy(cqv); + end_test(&f); + config.tear_down_data(&f); +} + +void bad_ping(grpc_end2end_test_config config) { + GPR_ASSERT(config.feature_mask & FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION); + test_bad_ping(config); +} + +void bad_ping_pre_init(void) {} diff --git a/test/core/end2end/tests/ping.c b/test/core/end2end/tests/ping.c index 082ac641f0b..471e02aaae4 100644 --- a/test/core/end2end/tests/ping.c +++ b/test/core/end2end/tests/ping.c @@ -52,16 +52,26 @@ static void test_ping(grpc_end2end_test_config config, grpc_connectivity_state state = GRPC_CHANNEL_IDLE; int i; - grpc_arg a[] = {{.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS, - .value.integer = min_time_between_pings_ms}, - {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 20}}; - grpc_channel_args client_args = {.num_args = GPR_ARRAY_SIZE(a), .args = a}; + grpc_arg client_a[] = {{.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MIN_TIME_BETWEEN_PINGS_MS, + .value.integer = 0}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, + .value.integer = 20}}; + grpc_arg server_a[] = { + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_S, + .value.integer = 0}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_KEEPALIVE_PERMIT_WITHOUT_CALLS, + .value.integer = 1}}; + grpc_channel_args client_args = {.num_args = GPR_ARRAY_SIZE(client_a), + .args = client_a}; + grpc_channel_args server_args = {.num_args = GPR_ARRAY_SIZE(server_a), + .args = server_a}; config.init_client(&f, &client_args); - config.init_server(&f, NULL); + config.init_server(&f, &server_args); grpc_channel_ping(f.client, f.cq, tag(0), NULL); CQ_EXPECT_COMPLETION(cqv, tag(0), 0); diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index be1d8768bd5..822d02f2126 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7119,6 +7119,7 @@ "test/core/end2end/end2end_tests.h", "test/core/end2end/tests/authority_not_supported.c", "test/core/end2end/tests/bad_hostname.c", + "test/core/end2end/tests/bad_ping.c", "test/core/end2end/tests/binary_metadata.c", "test/core/end2end/tests/call_creds.c", "test/core/end2end/tests/cancel_after_accept.c", @@ -7194,6 +7195,7 @@ "test/core/end2end/end2end_tests.h", "test/core/end2end/tests/authority_not_supported.c", "test/core/end2end/tests/bad_hostname.c", + "test/core/end2end/tests/bad_ping.c", "test/core/end2end/tests/binary_metadata.c", "test/core/end2end/tests/cancel_after_accept.c", "test/core/end2end/tests/cancel_after_client_done.c", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index b878bc12310..d40d5717f41 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -5725,6 +5725,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_census_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -6925,6 +6948,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -8123,6 +8169,28 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_fakesec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -10331,6 +10399,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -11523,6 +11614,25 @@ "linux" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_test", + "platforms": [ + "linux" + ] + }, { "args": [ "binary_metadata" @@ -12519,6 +12629,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -13675,6 +13808,30 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_http_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -14921,6 +15078,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_load_reporting_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -16123,6 +16303,30 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_oauth2_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -21707,6 +21911,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -22907,6 +23134,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_cert_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -24109,6 +24359,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_uds_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -25139,6 +25412,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_census_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -26312,6 +26608,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -28666,6 +28985,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -29701,6 +30043,25 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_nosec_test", + "platforms": [ + "linux" + ] + }, { "args": [ "binary_metadata" @@ -30870,6 +31231,29 @@ "linux" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -31847,6 +32231,30 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_http_proxy_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -32980,6 +33388,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_load_reporting_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -38481,6 +38912,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_uds_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj index 5a2d6acd563..e3adf793d63 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj @@ -159,6 +159,8 @@ + + diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters index 3a870b945e6..cfb8d043baf 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_nosec_tests/end2end_nosec_tests.vcxproj.filters @@ -13,6 +13,9 @@ test\core\end2end\tests + + test\core\end2end\tests + test\core\end2end\tests diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj index 4b001a751d8..a67f509e255 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj @@ -159,6 +159,8 @@ + + diff --git a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters index 2eace64a893..97ba77a42e1 100644 --- a/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters +++ b/vsprojects/vcxproj/test/end2end/tests/end2end_tests/end2end_tests.vcxproj.filters @@ -13,6 +13,9 @@ test\core\end2end\tests + + test\core\end2end\tests + test\core\end2end\tests From 114a848972e1243543f6eb7c4dc2be1d0867504a Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 22 Mar 2017 14:34:37 -0700 Subject: [PATCH 46/90] Disable bdp probe in bad_ping test --- src/core/ext/transport/chttp2/transport/chttp2_transport.c | 1 + test/core/end2end/tests/bad_ping.c | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index c4deb24e41e..37553831ff3 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1471,6 +1471,7 @@ static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, void grpc_chttp2_ping_strike(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t) { + gpr_log(GPR_DEBUG, "PING strike"); if (++t->ping_recv_state.ping_strikes > t->ping_policy.max_ping_strikes && t->ping_policy.max_ping_strikes != 0) { send_goaway(exec_ctx, t, diff --git a/test/core/end2end/tests/bad_ping.c b/test/core/end2end/tests/bad_ping.c index 97b02c90722..cdeefa4f952 100644 --- a/test/core/end2end/tests/bad_ping.c +++ b/test/core/end2end/tests/bad_ping.c @@ -85,7 +85,10 @@ static void test_bad_ping(grpc_end2end_test_config config) { .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, - .value.integer = 20}}; + .value.integer = 20}, + {.type = GRPC_ARG_INTEGER, + .key = GRPC_ARG_HTTP2_BDP_PROBE, + .value.integer = 0}}; grpc_arg server_a[] = { {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_S, From 78da25097d1ea4f1ccc55d244e3533eddf894915 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Fri, 24 Mar 2017 13:40:22 -0700 Subject: [PATCH 47/90] Update with the new grpc_erro interface --- .../ext/transport/chttp2/transport/chttp2_transport.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 37553831ff3..173964c92ee 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1475,11 +1475,12 @@ void grpc_chttp2_ping_strike(grpc_exec_ctx *exec_ctx, if (++t->ping_recv_state.ping_strikes > t->ping_policy.max_ping_strikes && t->ping_policy.max_ping_strikes != 0) { send_goaway(exec_ctx, t, - grpc_error_set_int(GRPC_ERROR_CREATE("too_many_pings"), - GRPC_ERROR_INT_HTTP2_ERROR, - GRPC_HTTP2_ENHANCE_YOUR_CALM)); + grpc_error_set_int( + GRPC_ERROR_CREATE_FROM_STATIC_STRING("too_many_pings"), + GRPC_ERROR_INT_HTTP2_ERROR, GRPC_HTTP2_ENHANCE_YOUR_CALM)); /*The transport will be closed after the write is done */ - close_transport_locked(exec_ctx, t, GRPC_ERROR_CREATE("Too many pings")); + close_transport_locked( + exec_ctx, t, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Too many pings")); } } From fe5934136026d612ff2b645a23b99d7991ce0380 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 27 Mar 2017 15:36:53 -0700 Subject: [PATCH 48/90] Reword comments --- .../ext/transport/chttp2/transport/chttp2_transport.c | 4 ++-- src/core/ext/transport/chttp2/transport/frame_ping.c | 8 ++++---- src/core/ext/transport/chttp2/transport/internal.h | 7 ++++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 173964c92ee..685e56efa87 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1469,8 +1469,8 @@ static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, GRPC_ERROR_UNREF(error); } -void grpc_chttp2_ping_strike(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t) { +void grpc_chttp2_add_ping_strike(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t) { gpr_log(GPR_DEBUG, "PING strike"); if (++t->ping_recv_state.ping_strikes > t->ping_policy.max_ping_strikes && t->ping_policy.max_ping_strikes != 0) { diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 61082b97441..6016e43127c 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -111,16 +111,16 @@ grpc_error *grpc_chttp2_ping_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, if (t->keepalive_permit_without_calls == 0 && grpc_chttp2_stream_map_size(&t->stream_map) == 0) { - /* The “2 hours” restricts the number of PINGS to an implementation - equivalent to TCP Keep-Alive, whose interval is specified to - default to no less than two hours in RFC1122. */ + /* According to RFC1122, the interval of TCP Keep-Alive is default to + no less than two hours. When there is no outstanding streams, we + restrict the number of PINGS equivalent to TCP Keep-Alive. */ next_allowed_ping = gpr_time_add(t->ping_recv_state.last_ping_recv_time, gpr_time_from_seconds(7200, GPR_TIMESPAN)); } if (gpr_time_cmp(next_allowed_ping, now) > 0) { - grpc_chttp2_ping_strike(exec_ctx, t); + grpc_chttp2_add_ping_strike(exec_ctx, t); } t->ping_recv_state.last_ping_recv_time = now; diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index c302d4ac348..d38edb5bfc2 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -800,7 +800,12 @@ void grpc_chttp2_incoming_byte_stream_finished( void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, uint64_t id); -void grpc_chttp2_ping_strike(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t); +/** Add a new ping strike to ping_recv_state.ping_strikes. If + ping_recv_state.ping_strikes > ping_policy.max_ping_strikes, it sends GOAWAY + with error code ENHANCE_YOUR_CALM and additional debug data resembling + “too_many_pings” followed by immediately closing the connection. */ +void grpc_chttp2_add_ping_strike(grpc_exec_ctx *exec_ctx, + grpc_chttp2_transport *t); typedef enum { /* don't initiate a transport write, but piggyback on the next one */ From c18d4b39c808d0507f0654fdf7ecad70a9a54945 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 29 Mar 2017 15:04:42 -0700 Subject: [PATCH 49/90] Change time unit to ms --- include/grpc/impl/codegen/grpc_types.h | 8 +-- .../chttp2/transport/chttp2_transport.c | 67 ++++++++++--------- test/core/end2end/tests/bad_ping.c | 4 +- test/core/end2end/tests/keepalive_timeout.c | 6 +- test/core/end2end/tests/ping.c | 4 +- 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index cb22785b354..07d883e3e2f 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -209,17 +209,17 @@ typedef struct { #define GRPC_ARG_HTTP2_MAX_PING_STRIKES "grpc.http2.max_ping_strikes" /** Minimum allowed time between two pings without sending any data frame. Int valued, seconds */ -#define GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_S \ - "grpc.http2.min_ping_interval_without_data" +#define GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS \ + "grpc.http2.min_ping_interval_without_data_ms" /** How much data are we willing to queue up per stream if GRPC_WRITE_BUFFER_HINT is set? This is an upper bound */ #define GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE "grpc.http2.write_buffer_size" /** After a duration of this time the client pings the server to see if the transport is still alive. Int valued, seconds. */ -#define GRPC_ARG_CLIENT_KEEPALIVE_TIME_S "grpc.client_keepalive_time" +#define GRPC_ARG_CLIENT_KEEPALIVE_TIME_MS "grpc.client_keepalive_time_ms" /** After waiting for a duration of this time, if the client does not receive the ping ack, it will close the transport. Int valued, seconds. */ -#define GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_S "grpc.client_keepalive_timeout" +#define GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_MS "grpc.client_keepalive_timeout_ms" /** Is it permissible to send keepalive pings without any outstanding streams. Int valued, 0(false)/1(true). */ #define GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS \ diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 685e56efa87..366cf7f2caa 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -69,13 +69,14 @@ #define MAX_WRITE_BUFFER_SIZE (64 * 1024 * 1024) #define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024) -#define DEFAULT_CLIENT_KEEPALIVE_TIME_S INT_MAX -#define DEFAULT_CLIENT_KEEPALIVE_TIMEOUT_S 20 +#define DEFAULT_CLIENT_KEEPALIVE_TIME_MS INT_MAX +#define DEFAULT_CLIENT_KEEPALIVE_TIMEOUT_MS 20000 #define DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS false -static int g_default_client_keepalive_time_s = DEFAULT_CLIENT_KEEPALIVE_TIME_S; -static int g_default_client_keepalive_timeout_s = - DEFAULT_CLIENT_KEEPALIVE_TIMEOUT_S; +static int g_default_client_keepalive_time_ms = + DEFAULT_CLIENT_KEEPALIVE_TIME_MS; +static int g_default_client_keepalive_timeout_ms = + DEFAULT_CLIENT_KEEPALIVE_TIMEOUT_MS; static bool g_default_keepalive_permit_without_calls = DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS; @@ -154,7 +155,7 @@ static void retry_initiate_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, #define DEFAULT_MIN_TIME_BETWEEN_PINGS_MS 0 #define DEFAULT_MAX_PINGS_BETWEEN_DATA 3 #define DEFAULT_MAX_PING_STRIKES 2 -#define DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_S 300 +#define DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS 300000 /* 5 minutes */ /** keepalive-relevant functions */ static void init_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, @@ -354,21 +355,21 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, .min_time_between_pings = gpr_time_from_millis(DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, GPR_TIMESPAN), .max_ping_strikes = DEFAULT_MAX_PING_STRIKES, - .min_ping_interval_without_data = gpr_time_from_seconds( - DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_S, GPR_TIMESPAN), + .min_ping_interval_without_data = gpr_time_from_millis( + DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), }; /* client-side keepalive setting */ t->keepalive_time = - g_default_client_keepalive_time_s == INT_MAX + g_default_client_keepalive_time_ms == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_seconds(g_default_client_keepalive_time_s, - GPR_TIMESPAN); + : gpr_time_from_millis(g_default_client_keepalive_time_ms, + GPR_TIMESPAN); t->keepalive_timeout = - g_default_client_keepalive_timeout_s == INT_MAX + g_default_client_keepalive_timeout_ms == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_seconds(g_default_client_keepalive_timeout_s, - GPR_TIMESPAN); + : gpr_time_from_millis(g_default_client_keepalive_timeout_ms, + GPR_TIMESPAN); t->keepalive_permit_without_calls = g_default_keepalive_permit_without_calls; if (channel_args) { @@ -414,13 +415,14 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, (grpc_integer_options){DEFAULT_MIN_TIME_BETWEEN_PINGS_MS, 0, INT_MAX}), GPR_TIMESPAN); - } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_S)) { - t->ping_policy.min_ping_interval_without_data = gpr_time_from_seconds( + } else if (0 == + strcmp(channel_args->args[i].key, + GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS)) { + t->ping_policy.min_ping_interval_without_data = gpr_time_from_millis( grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_S, - 0, INT_MAX}), + (grpc_integer_options){ + DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, 0, INT_MAX}), GPR_TIMESPAN); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE)) { @@ -432,23 +434,23 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->enable_bdp_probe = grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){1, 0, 1}); } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_CLIENT_KEEPALIVE_TIME_S)) { + GRPC_ARG_CLIENT_KEEPALIVE_TIME_MS)) { const int value = grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){g_default_client_keepalive_time_s, 1, + (grpc_integer_options){g_default_client_keepalive_time_ms, 1, INT_MAX}); t->keepalive_time = value == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_seconds(value, GPR_TIMESPAN); + : gpr_time_from_millis(value, GPR_TIMESPAN); } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_S)) { + GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_MS)) { const int value = grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){g_default_client_keepalive_timeout_s, 0, + (grpc_integer_options){g_default_client_keepalive_timeout_ms, 0, INT_MAX}); t->keepalive_timeout = value == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_seconds(value, GPR_TIMESPAN); + : gpr_time_from_millis(value, GPR_TIMESPAN); } else if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS)) { t->keepalive_permit_without_calls = @@ -2172,15 +2174,16 @@ void grpc_chttp2_config_default_keepalive_args(grpc_channel_args *args) { size_t i; if (args) { for (i = 0; i < args->num_args; i++) { - if (0 == strcmp(args->args[i].key, GRPC_ARG_CLIENT_KEEPALIVE_TIME_S)) { - g_default_client_keepalive_time_s = grpc_channel_arg_get_integer( - &args->args[i], (grpc_integer_options){ - g_default_client_keepalive_time_s, 1, INT_MAX}); + if (0 == strcmp(args->args[i].key, GRPC_ARG_CLIENT_KEEPALIVE_TIME_MS)) { + g_default_client_keepalive_time_ms = grpc_channel_arg_get_integer( + &args->args[i], + (grpc_integer_options){g_default_client_keepalive_time_ms, 1, + INT_MAX}); } else if (0 == strcmp(args->args[i].key, - GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_S)) { - g_default_client_keepalive_timeout_s = grpc_channel_arg_get_integer( + GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_MS)) { + g_default_client_keepalive_timeout_ms = grpc_channel_arg_get_integer( &args->args[i], - (grpc_integer_options){g_default_client_keepalive_timeout_s, 0, + (grpc_integer_options){g_default_client_keepalive_timeout_ms, 0, INT_MAX}); } else if (0 == strcmp(args->args[i].key, GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS)) { diff --git a/test/core/end2end/tests/bad_ping.c b/test/core/end2end/tests/bad_ping.c index cdeefa4f952..01a6aeaa04f 100644 --- a/test/core/end2end/tests/bad_ping.c +++ b/test/core/end2end/tests/bad_ping.c @@ -91,8 +91,8 @@ static void test_bad_ping(grpc_end2end_test_config config) { .value.integer = 0}}; grpc_arg server_a[] = { {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_S, - .value.integer = 300}, + .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS, + .value.integer = 300000 /* 5 minutes */}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_MAX_PING_STRIKES, .value.integer = MAX_PING_STRIKES}}; diff --git a/test/core/end2end/tests/keepalive_timeout.c b/test/core/end2end/tests/keepalive_timeout.c index 44b6e12abc7..d492585b68b 100644 --- a/test/core/end2end/tests/keepalive_timeout.c +++ b/test/core/end2end/tests/keepalive_timeout.c @@ -111,10 +111,10 @@ static void test_keepalive_timeout(grpc_end2end_test_config config) { gpr_timespec deadline = five_seconds_time(); grpc_arg keepalive_args[] = {{.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_CLIENT_KEEPALIVE_TIME_S, - .value.integer = 2}, + .key = GRPC_ARG_CLIENT_KEEPALIVE_TIME_MS, + .value.integer = 1500}, {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_S, + .key = GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_MS, .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_BDP_PROBE, diff --git a/test/core/end2end/tests/ping.c b/test/core/end2end/tests/ping.c index 471e02aaae4..c51c4993e25 100644 --- a/test/core/end2end/tests/ping.c +++ b/test/core/end2end/tests/ping.c @@ -60,10 +60,10 @@ static void test_ping(grpc_end2end_test_config config, .value.integer = 20}}; grpc_arg server_a[] = { {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_S, + .key = GRPC_ARG_HTTP2_MIN_PING_INTERVAL_WITHOUT_DATA_MS, .value.integer = 0}, {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_HTTP2_KEEPALIVE_PERMIT_WITHOUT_CALLS, + .key = GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, .value.integer = 1}}; grpc_channel_args client_args = {.num_args = GPR_ARRAY_SIZE(client_a), .args = client_a}; From 5f89996e69414397a522a149b477139d9d3aadf5 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 29 Mar 2017 17:55:28 -0700 Subject: [PATCH 50/90] Enable server-side keepalive pings --- include/grpc/impl/codegen/grpc_types.h | 13 +-- .../chttp2/transport/chttp2_transport.c | 86 +++++++++++++------ .../ext/transport/chttp2/transport/internal.h | 3 +- test/core/end2end/tests/keepalive_timeout.c | 4 +- 4 files changed, 71 insertions(+), 35 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 07d883e3e2f..204763cd536 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -214,12 +214,13 @@ typedef struct { /** How much data are we willing to queue up per stream if GRPC_WRITE_BUFFER_HINT is set? This is an upper bound */ #define GRPC_ARG_HTTP2_WRITE_BUFFER_SIZE "grpc.http2.write_buffer_size" -/** After a duration of this time the client pings the server to see if the - transport is still alive. Int valued, seconds. */ -#define GRPC_ARG_CLIENT_KEEPALIVE_TIME_MS "grpc.client_keepalive_time_ms" -/** After waiting for a duration of this time, if the client does not receive - the ping ack, it will close the transport. Int valued, seconds. */ -#define GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_MS "grpc.client_keepalive_timeout_ms" +/** After a duration of this time the client/server pings its peer to see if the + transport is still alive. Int valued, milliseconds. */ +#define GRPC_ARG_KEEPALIVE_TIME_MS "grpc.keepalive_time_ms" +/** After waiting for a duration of this time, if the keepalive ping sender does + not receive the ping ack, it will close the transport. Int valued, + milliseconds. */ +#define GRPC_ARG_KEEPALIVE_TIMEOUT_MS "grpc.keepalive_timeout_ms" /** Is it permissible to send keepalive pings without any outstanding streams. Int valued, 0(false)/1(true). */ #define GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS \ diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 366cf7f2caa..8cdcc29f4ee 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -70,13 +70,19 @@ #define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024) #define DEFAULT_CLIENT_KEEPALIVE_TIME_MS INT_MAX -#define DEFAULT_CLIENT_KEEPALIVE_TIMEOUT_MS 20000 +#define DEFAULT_CLIENT_KEEPALIVE_TIMEOUT_MS 20000 /* 20 seconds */ +#define DEFAULT_SERVER_KEEPALIVE_TIME_MS 7200000 /* 2 hours */ +#define DEFAULT_SERVER_KEEPALIVE_TIMEOUT_MS 20000 /* 20 seconds */ #define DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS false static int g_default_client_keepalive_time_ms = DEFAULT_CLIENT_KEEPALIVE_TIME_MS; static int g_default_client_keepalive_timeout_ms = DEFAULT_CLIENT_KEEPALIVE_TIMEOUT_MS; +static int g_default_server_keepalive_time_ms = + DEFAULT_SERVER_KEEPALIVE_TIME_MS; +static int g_default_server_keepalive_timeout_ms = + DEFAULT_SERVER_KEEPALIVE_TIMEOUT_MS; static bool g_default_keepalive_permit_without_calls = DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS; @@ -359,17 +365,30 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, DEFAULT_MIN_PING_INTERVAL_WITHOUT_DATA_MS, GPR_TIMESPAN), }; - /* client-side keepalive setting */ - t->keepalive_time = - g_default_client_keepalive_time_ms == INT_MAX - ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(g_default_client_keepalive_time_ms, - GPR_TIMESPAN); - t->keepalive_timeout = - g_default_client_keepalive_timeout_ms == INT_MAX - ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(g_default_client_keepalive_timeout_ms, - GPR_TIMESPAN); + /* Keepalive setting */ + if (t->is_client) { + t->keepalive_time = + g_default_client_keepalive_time_ms == INT_MAX + ? gpr_inf_future(GPR_TIMESPAN) + : gpr_time_from_millis(g_default_client_keepalive_time_ms, + GPR_TIMESPAN); + t->keepalive_timeout = + g_default_client_keepalive_timeout_ms == INT_MAX + ? gpr_inf_future(GPR_TIMESPAN) + : gpr_time_from_millis(g_default_client_keepalive_timeout_ms, + GPR_TIMESPAN); + } else { + t->keepalive_time = + g_default_server_keepalive_time_ms == INT_MAX + ? gpr_inf_future(GPR_TIMESPAN) + : gpr_time_from_millis(g_default_server_keepalive_time_ms, + GPR_TIMESPAN); + t->keepalive_timeout = + g_default_server_keepalive_timeout_ms == INT_MAX + ? gpr_inf_future(GPR_TIMESPAN) + : gpr_time_from_millis(g_default_server_keepalive_timeout_ms, + GPR_TIMESPAN); + } t->keepalive_permit_without_calls = g_default_keepalive_permit_without_calls; if (channel_args) { @@ -434,20 +453,24 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->enable_bdp_probe = grpc_channel_arg_get_integer( &channel_args->args[i], (grpc_integer_options){1, 0, 1}); } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_CLIENT_KEEPALIVE_TIME_MS)) { + GRPC_ARG_KEEPALIVE_TIME_MS)) { const int value = grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){g_default_client_keepalive_time_ms, 1, - INT_MAX}); + (grpc_integer_options){t->is_client + ? g_default_client_keepalive_time_ms + : g_default_server_keepalive_time_ms, + 1, INT_MAX}); t->keepalive_time = value == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) : gpr_time_from_millis(value, GPR_TIMESPAN); } else if (0 == strcmp(channel_args->args[i].key, - GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_MS)) { + GRPC_ARG_KEEPALIVE_TIMEOUT_MS)) { const int value = grpc_channel_arg_get_integer( &channel_args->args[i], - (grpc_integer_options){g_default_client_keepalive_timeout_ms, 0, - INT_MAX}); + (grpc_integer_options){t->is_client + ? g_default_client_keepalive_timeout_ms + : g_default_server_keepalive_timeout_ms, + 0, INT_MAX}); t->keepalive_timeout = value == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) : gpr_time_from_millis(value, GPR_TIMESPAN); @@ -511,8 +534,8 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->ping_recv_state.last_ping_recv_time = gpr_inf_past(GPR_CLOCK_MONOTONIC); t->ping_recv_state.ping_strikes = 0; - /* Start client-side keepalive pings */ - if (t->is_client) { + /* Start keepalive pings */ + if (gpr_time_cmp(t->keepalive_time, gpr_inf_future(GPR_TIMESPAN)) != 0) { t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_WAITING; GRPC_CHTTP2_REF_TRANSPORT(t, "init keepalive ping"); grpc_timer_init( @@ -2170,21 +2193,32 @@ static void finish_bdp_ping_locked(grpc_exec_ctx *exec_ctx, void *tp, GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "bdp_ping"); } -void grpc_chttp2_config_default_keepalive_args(grpc_channel_args *args) { +void grpc_chttp2_config_default_keepalive_args(grpc_channel_args *args, + bool is_client) { size_t i; if (args) { for (i = 0; i < args->num_args; i++) { - if (0 == strcmp(args->args[i].key, GRPC_ARG_CLIENT_KEEPALIVE_TIME_MS)) { - g_default_client_keepalive_time_ms = grpc_channel_arg_get_integer( + if (0 == strcmp(args->args[i].key, GRPC_ARG_KEEPALIVE_TIME_MS)) { + const int value = grpc_channel_arg_get_integer( &args->args[i], (grpc_integer_options){g_default_client_keepalive_time_ms, 1, INT_MAX}); - } else if (0 == strcmp(args->args[i].key, - GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_MS)) { - g_default_client_keepalive_timeout_ms = grpc_channel_arg_get_integer( + if (is_client) { + g_default_client_keepalive_time_ms = value; + } else { + g_default_server_keepalive_time_ms = value; + } + } else if (0 == + strcmp(args->args[i].key, GRPC_ARG_KEEPALIVE_TIMEOUT_MS)) { + const int value = grpc_channel_arg_get_integer( &args->args[i], (grpc_integer_options){g_default_client_keepalive_timeout_ms, 0, INT_MAX}); + if (is_client) { + g_default_client_keepalive_timeout_ms = value; + } else { + g_default_server_keepalive_timeout_ms = value; + } } else if (0 == strcmp(args->args[i].key, GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS)) { g_default_keepalive_permit_without_calls = diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index d38edb5bfc2..6eb848b8d77 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -846,6 +846,7 @@ uint32_t grpc_chttp2_target_incoming_window(grpc_chttp2_transport *t); /** Set the default keepalive configurations, must only be called at initialization */ -void grpc_chttp2_config_default_keepalive_args(grpc_channel_args *args); +void grpc_chttp2_config_default_keepalive_args(grpc_channel_args *args, + bool is_client); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INTERNAL_H */ diff --git a/test/core/end2end/tests/keepalive_timeout.c b/test/core/end2end/tests/keepalive_timeout.c index d492585b68b..4b377559599 100644 --- a/test/core/end2end/tests/keepalive_timeout.c +++ b/test/core/end2end/tests/keepalive_timeout.c @@ -111,10 +111,10 @@ static void test_keepalive_timeout(grpc_end2end_test_config config) { gpr_timespec deadline = five_seconds_time(); grpc_arg keepalive_args[] = {{.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_CLIENT_KEEPALIVE_TIME_MS, + .key = GRPC_ARG_KEEPALIVE_TIME_MS, .value.integer = 1500}, {.type = GRPC_ARG_INTEGER, - .key = GRPC_ARG_CLIENT_KEEPALIVE_TIMEOUT_MS, + .key = GRPC_ARG_KEEPALIVE_TIMEOUT_MS, .value.integer = 0}, {.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_HTTP2_BDP_PROBE, From ebe6baaf64237b84e9df90891355ab5bf9e678c6 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Fri, 31 Mar 2017 16:28:46 -0700 Subject: [PATCH 51/90] Double keepalive_time when the client needs to be calmed --- .../chttp2/transport/chttp2_transport.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 8cdcc29f4ee..1de36b91060 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -964,6 +964,25 @@ void grpc_chttp2_add_incoming_goaway(grpc_exec_ctx *exec_ctx, // GRPC_CHTTP2_IF_TRACING( // gpr_log(GPR_DEBUG, "got goaway [%d]: %s", goaway_error, msg)); t->seen_goaway = 1; + + /* When a client receives a GOAWAY with error code ENHANCE_YOUR_CALM and debug + * data equal to “too_many_pings”, it should log the occurrence at a log level + * that is enabled by default and double the configured KEEPALIVE_TIME used + * for new connections on that channel. */ + if (t->is_client && goaway_error == GRPC_HTTP2_ENHANCE_YOUR_CALM && + grpc_slice_str_cmp(goaway_text, "too_many_pings") == 0) { + gpr_log(GPR_ERROR, + "Received a GOAWAY with error code ENHANCE_YOUR_CALM and debug data" + "equal to \"too_many_pings\""); + double current_keepalive_time_ms = + gpr_timespec_to_micros(t->keepalive_time) / 1000; + t->keepalive_time = + current_keepalive_time_ms > INT_MAX / 2 + ? gpr_inf_future(GPR_TIMESPAN) + : gpr_time_from_millis((int64_t)(current_keepalive_time_ms * 2), + GPR_TIMESPAN); + } + /* lie: use transient failure from the transport to indicate goaway has been * received */ connectivity_state_set( From 8b0a4d207c3c7bcc0651d3d9c0e0d5eef493f829 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 3 Apr 2017 11:24:42 -0700 Subject: [PATCH 52/90] Add KEEPALIVE_TIME_BACKOFF_MULTIPLIER macro --- .../ext/transport/chttp2/transport/chttp2_transport.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 1de36b91060..1ece5294a3e 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -74,6 +74,7 @@ #define DEFAULT_SERVER_KEEPALIVE_TIME_MS 7200000 /* 2 hours */ #define DEFAULT_SERVER_KEEPALIVE_TIMEOUT_MS 20000 /* 20 seconds */ #define DEFAULT_KEEPALIVE_PERMIT_WITHOUT_CALLS false +#define KEEPALIVE_TIME_BACKOFF_MULTIPLIER 2 static int g_default_client_keepalive_time_ms = DEFAULT_CLIENT_KEEPALIVE_TIME_MS; @@ -972,14 +973,15 @@ void grpc_chttp2_add_incoming_goaway(grpc_exec_ctx *exec_ctx, if (t->is_client && goaway_error == GRPC_HTTP2_ENHANCE_YOUR_CALM && grpc_slice_str_cmp(goaway_text, "too_many_pings") == 0) { gpr_log(GPR_ERROR, - "Received a GOAWAY with error code ENHANCE_YOUR_CALM and debug data" - "equal to \"too_many_pings\""); + "Received a GOAWAY with error code ENHANCE_YOUR_CALM and debug " + "data equal to \"too_many_pings\""); double current_keepalive_time_ms = gpr_timespec_to_micros(t->keepalive_time) / 1000; t->keepalive_time = - current_keepalive_time_ms > INT_MAX / 2 + current_keepalive_time_ms > INT_MAX / KEEPALIVE_TIME_BACKOFF_MULTIPLIER ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis((int64_t)(current_keepalive_time_ms * 2), + : gpr_time_from_millis((int64_t)(current_keepalive_time_ms * + KEEPALIVE_TIME_BACKOFF_MULTIPLIER), GPR_TIMESPAN); } From ed2a5470dbf2154aa3bd6ed44ad6151d557b4446 Mon Sep 17 00:00:00 2001 From: Chris Evans Date: Mon, 27 Mar 2017 17:34:51 -0500 Subject: [PATCH 53/90] Calculate RPC deadline right before call creation Most core end2end tests are written around a single RPC with an explicit deadline. Currently, this deadline is calculated before bringing up the test fixture, and thus time spent bringing up the test fixture is taken from the time allotted to the RPC. This change moves the calculation of these deadlines immediately before call creation. Additionally, the functions that calculate these deadlines have been renamed from 'xx_seconds_time()' to 'xx_seconds_from_now()' for clarity. --- .../end2end/tests/authority_not_supported.c | 10 ++++++---- test/core/end2end/tests/bad_hostname.c | 10 ++++++---- test/core/end2end/tests/binary_metadata.c | 10 ++++++---- test/core/end2end/tests/call_creds.c | 12 ++++++----- test/core/end2end/tests/cancel_after_accept.c | 14 +++++++------ .../end2end/tests/cancel_after_client_done.c | 10 ++++++---- test/core/end2end/tests/cancel_after_invoke.c | 10 ++++++---- .../core/end2end/tests/cancel_before_invoke.c | 10 ++++++---- test/core/end2end/tests/cancel_in_a_vacuum.c | 10 ++++++---- test/core/end2end/tests/cancel_with_status.c | 10 ++++++---- test/core/end2end/tests/compressed_payload.c | 12 ++++++----- test/core/end2end/tests/default_host.c | 10 ++++++---- test/core/end2end/tests/disappearing_server.c | 10 ++++++---- test/core/end2end/tests/empty_batch.c | 10 ++++++---- .../end2end/tests/filter_call_init_fails.c | 10 ++++++---- test/core/end2end/tests/filter_causes_close.c | 10 ++++++---- test/core/end2end/tests/filter_latency.c | 10 ++++++---- .../end2end/tests/graceful_server_shutdown.c | 10 ++++++---- test/core/end2end/tests/high_initial_seqno.c | 10 ++++++---- test/core/end2end/tests/hpack_size.c | 10 ++++++---- test/core/end2end/tests/idempotent_request.c | 10 ++++++---- .../core/end2end/tests/invoke_large_request.c | 6 +++--- test/core/end2end/tests/keepalive_timeout.c | 16 ++++++++------- test/core/end2end/tests/large_metadata.c | 10 ++++++---- test/core/end2end/tests/load_reporting_hook.c | 10 ++++++---- .../end2end/tests/max_concurrent_streams.c | 20 ++++++++++--------- test/core/end2end/tests/max_message_length.c | 8 +++++--- test/core/end2end/tests/negative_deadline.c | 10 ++++++---- .../end2end/tests/network_status_change.c | 10 ++++++---- test/core/end2end/tests/no_logging.c | 10 ++++++---- test/core/end2end/tests/no_op.c | 8 +++++--- test/core/end2end/tests/payload.c | 10 ++++++---- test/core/end2end/tests/ping_pong_streaming.c | 10 ++++++---- test/core/end2end/tests/registered_call.c | 10 ++++++---- test/core/end2end/tests/request_with_flags.c | 10 ++++++---- .../core/end2end/tests/request_with_payload.c | 10 ++++++---- .../end2end/tests/resource_quota_server.c | 13 +++++++----- .../end2end/tests/server_finishes_request.c | 10 ++++++---- .../end2end/tests/shutdown_finishes_calls.c | 10 ++++++---- .../end2end/tests/shutdown_finishes_tags.c | 8 +++++--- .../end2end/tests/simple_cacheable_request.c | 10 ++++++---- .../end2end/tests/simple_delayed_request.c | 10 ++++++---- test/core/end2end/tests/simple_metadata.c | 10 ++++++---- test/core/end2end/tests/simple_request.c | 10 ++++++---- .../end2end/tests/streaming_error_response.c | 10 ++++++---- test/core/end2end/tests/trailing_metadata.c | 10 ++++++---- test/core/end2end/tests/write_buffering.c | 10 ++++++---- .../end2end/tests/write_buffering_at_end.c | 10 ++++++---- 48 files changed, 296 insertions(+), 201 deletions(-) diff --git a/test/core/end2end/tests/authority_not_supported.c b/test/core/end2end/tests/authority_not_supported.c index 7db2fd6b278..192a20d4007 100644 --- a/test/core/end2end/tests/authority_not_supported.c +++ b/test/core/end2end/tests/authority_not_supported.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -102,7 +104,6 @@ static void test_with_authority_header(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_metadata meta_c[2] = {{grpc_slice_from_static_string("key1"), grpc_slice_from_static_string("val1"), 0, @@ -124,6 +125,7 @@ static void test_with_authority_header(grpc_end2end_test_config config) { grpc_slice details; grpc_slice host = grpc_slice_from_static_string("foo.test.google.fr"); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call(f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), &host, deadline, NULL); diff --git a/test/core/end2end/tests/bad_hostname.c b/test/core/end2end/tests/bad_hostname.c index f50a5805a23..c63c9c78ada 100644 --- a/test/core/end2end/tests/bad_hostname.c +++ b/test/core/end2end/tests/bad_hostname.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -99,7 +101,6 @@ static void end_test(grpc_end2end_test_fixture *f) { static void simple_request_body(grpc_end2end_test_fixture f) { grpc_call *c; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -112,6 +113,7 @@ static void simple_request_body(grpc_end2end_test_fixture f) { grpc_slice details; grpc_slice host = grpc_slice_from_static_string("slartibartfast.local"); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call(f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), &host, deadline, NULL); diff --git a/test/core/end2end/tests/binary_metadata.c b/test/core/end2end/tests/binary_metadata.c index 7fb60f4495b..9cb17c9d429 100644 --- a/test/core/end2end/tests/binary_metadata.c +++ b/test/core/end2end/tests/binary_metadata.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -108,7 +110,6 @@ static void test_request_response_with_metadata_and_payload( grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_byte_buffer *response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_metadata meta_c[2] = { {grpc_slice_from_static_string("key1-bin"), grpc_slice_from_static_string( @@ -147,6 +148,7 @@ static void test_request_response_with_metadata_and_payload( grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/call_creds.c b/test/core/end2end/tests/call_creds.c index 38cba50e129..d65073fbbe0 100644 --- a/test/core/end2end/tests/call_creds.c +++ b/test/core/end2end/tests/call_creds.c @@ -75,16 +75,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -143,7 +145,6 @@ static void request_response_with_payload_and_call_creds( grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_byte_buffer *response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f; cq_verifier *cqv; grpc_op ops[6]; @@ -165,6 +166,7 @@ static void request_response_with_payload_and_call_creds( f = begin_test(config, test_name, 0); cqv = cq_verifier_create(f.cq); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), @@ -383,7 +385,7 @@ static void test_request_with_server_rejecting_client_creds( grpc_op *op; grpc_call *c; grpc_end2end_test_fixture f; - gpr_timespec deadline = five_seconds_time(); + gpr_timespec deadline = five_seconds_from_now(); cq_verifier *cqv; grpc_metadata_array initial_metadata_recv; grpc_metadata_array trailing_metadata_recv; diff --git a/test/core/end2end/tests/cancel_after_accept.c b/test/core/end2end/tests/cancel_after_accept.c index 1a92aa48378..bda061ccb84 100644 --- a/test/core/end2end/tests/cancel_after_accept.c +++ b/test/core/end2end/tests/cancel_after_accept.c @@ -64,16 +64,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -110,9 +112,6 @@ static void test_cancel_after_accept(grpc_end2end_test_config config, grpc_op *op; grpc_call *c; grpc_call *s; - gpr_timespec deadline = use_service_config - ? gpr_inf_future(GPR_CLOCK_MONOTONIC) - : five_seconds_time(); grpc_metadata_array initial_metadata_recv; grpc_metadata_array trailing_metadata_recv; grpc_metadata_array request_metadata_recv; @@ -153,6 +152,9 @@ static void test_cancel_after_accept(grpc_end2end_test_config config, begin_test(config, "cancel_after_accept", args, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); + gpr_timespec deadline = use_service_config + ? gpr_inf_future(GPR_CLOCK_MONOTONIC) + : five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/service/method"), diff --git a/test/core/end2end/tests/cancel_after_client_done.c b/test/core/end2end/tests/cancel_after_client_done.c index 0afeecb037b..0e7291116fe 100644 --- a/test/core/end2end/tests/cancel_after_client_done.c +++ b/test/core/end2end/tests/cancel_after_client_done.c @@ -58,16 +58,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -105,7 +107,6 @@ static void test_cancel_after_accept_and_writes_closed( grpc_call *s; grpc_end2end_test_fixture f = begin_test( config, "test_cancel_after_accept_and_writes_closed", NULL, NULL); - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_metadata_array initial_metadata_recv; grpc_metadata_array trailing_metadata_recv; @@ -126,6 +127,7 @@ static void test_cancel_after_accept_and_writes_closed( grpc_raw_byte_buffer_create(&response_payload_slice, 1); int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/cancel_after_invoke.c b/test/core/end2end/tests/cancel_after_invoke.c index 8a96ef2f894..0ee59806fe1 100644 --- a/test/core/end2end/tests/cancel_after_invoke.c +++ b/test/core/end2end/tests/cancel_after_invoke.c @@ -60,16 +60,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -106,7 +108,6 @@ static void test_cancel_after_invoke(grpc_end2end_test_config config, grpc_call *c; grpc_end2end_test_fixture f = begin_test(config, "test_cancel_after_invoke", mode, NULL, NULL); - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_metadata_array initial_metadata_recv; grpc_metadata_array trailing_metadata_recv; @@ -121,6 +122,7 @@ static void test_cancel_after_invoke(grpc_end2end_test_config config, grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/cancel_before_invoke.c b/test/core/end2end/tests/cancel_before_invoke.c index 586aa7ead37..7d53091a1a7 100644 --- a/test/core/end2end/tests/cancel_before_invoke.c +++ b/test/core/end2end/tests/cancel_before_invoke.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -103,7 +105,6 @@ static void test_cancel_before_invoke(grpc_end2end_test_config config, grpc_call *c; grpc_end2end_test_fixture f = begin_test(config, "cancel_before_invoke", NULL, NULL); - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_metadata_array initial_metadata_recv; grpc_metadata_array trailing_metadata_recv; @@ -118,6 +119,7 @@ static void test_cancel_before_invoke(grpc_end2end_test_config config, grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/cancel_in_a_vacuum.c b/test/core/end2end/tests/cancel_in_a_vacuum.c index bc462ddcf56..c13fc88c532 100644 --- a/test/core/end2end/tests/cancel_in_a_vacuum.c +++ b/test/core/end2end/tests/cancel_in_a_vacuum.c @@ -58,16 +58,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -102,9 +104,9 @@ static void test_cancel_in_a_vacuum(grpc_end2end_test_config config, grpc_call *c; grpc_end2end_test_fixture f = begin_test(config, "test_cancel_in_a_vacuum", NULL, NULL); - gpr_timespec deadline = five_seconds_time(); cq_verifier *v_client = cq_verifier_create(f.cq); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/cancel_with_status.c b/test/core/end2end/tests/cancel_with_status.c index 7d03fe5580f..1e0e33b4e2f 100644 --- a/test/core/end2end/tests/cancel_with_status.c +++ b/test/core/end2end/tests/cancel_with_status.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -100,7 +102,6 @@ static void end_test(grpc_end2end_test_fixture *f) { static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f, size_t num_ops) { grpc_call *c; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -112,6 +113,7 @@ static void simple_request_body(grpc_end2end_test_config config, gpr_log(GPR_DEBUG, "test with %" PRIuPTR " ops", num_ops); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/compressed_payload.c b/test/core/end2end/tests/compressed_payload.c index 7dd8c112d11..c5b750af528 100644 --- a/test/core/end2end/tests/compressed_payload.c +++ b/test/core/end2end/tests/compressed_payload.c @@ -65,16 +65,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -113,7 +115,6 @@ static void request_for_disabled_algorithm( grpc_call *s; grpc_slice request_payload_slice; grpc_byte_buffer *request_payload; - gpr_timespec deadline = five_seconds_time(); grpc_channel_args *client_args; grpc_channel_args *server_args; grpc_end2end_test_fixture f; @@ -150,6 +151,7 @@ static void request_for_disabled_algorithm( f = begin_test(config, test_name, client_args, server_args); cqv = cq_verifier_create(f.cq); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), @@ -290,7 +292,6 @@ static void request_with_payload_template( grpc_call *s; grpc_slice request_payload_slice; grpc_byte_buffer *request_payload; - gpr_timespec deadline = five_seconds_time(); grpc_channel_args *client_args; grpc_channel_args *server_args; grpc_end2end_test_fixture f; @@ -329,6 +330,7 @@ static void request_with_payload_template( f = begin_test(config, test_name, client_args, server_args); cqv = cq_verifier_create(f.cq); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/default_host.c b/test/core/end2end/tests/default_host.c index bc1829b24b7..8dccca20c7c 100644 --- a/test/core/end2end/tests/default_host.c +++ b/test/core/end2end/tests/default_host.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -100,7 +102,6 @@ static void end_test(grpc_end2end_test_fixture *f) { static void simple_request_body(grpc_end2end_test_fixture f) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -114,6 +115,7 @@ static void simple_request_body(grpc_end2end_test_fixture f) { int was_cancelled = 2; char *peer; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call(f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), NULL, deadline, NULL); diff --git a/test/core/end2end/tests/disappearing_server.c b/test/core/end2end/tests/disappearing_server.c index 05440a6f56c..e2b888a3bbd 100644 --- a/test/core/end2end/tests/disappearing_server.c +++ b/test/core/end2end/tests/disappearing_server.c @@ -45,16 +45,18 @@ static void *tag(intptr_t t) { return (void *)t; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -84,7 +86,6 @@ static void do_request_and_shutdown_server(grpc_end2end_test_config config, cq_verifier *cqv) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); grpc_op ops[6]; grpc_op *op; grpc_metadata_array initial_metadata_recv; @@ -96,6 +97,7 @@ static void do_request_and_shutdown_server(grpc_end2end_test_config config, grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f->client, NULL, GRPC_PROPAGATE_DEFAULTS, f->cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/empty_batch.c b/test/core/end2end/tests/empty_batch.c index 50bb6b849e1..37ca81384f3 100644 --- a/test/core/end2end/tests/empty_batch.c +++ b/test/core/end2end/tests/empty_batch.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -100,11 +102,11 @@ static void end_test(grpc_end2end_test_fixture *f) { static void empty_batch_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f) { grpc_call *c; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_call_error error; grpc_op *op = NULL; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/filter_call_init_fails.c b/test/core/end2end/tests/filter_call_init_fails.c index ebfe3b03dc3..ffeb2715c5b 100644 --- a/test/core/end2end/tests/filter_call_init_fails.c +++ b/test/core/end2end/tests/filter_call_init_fails.c @@ -65,16 +65,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -112,7 +114,6 @@ static void test_request(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test(config, "filter_call_init_fails", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -127,6 +128,7 @@ static void test_request(grpc_end2end_test_config config) { grpc_call_error error; grpc_slice details; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/filter_causes_close.c b/test/core/end2end/tests/filter_causes_close.c index 62bf20809c3..9115823a743 100644 --- a/test/core/end2end/tests/filter_causes_close.c +++ b/test/core/end2end/tests/filter_causes_close.c @@ -62,16 +62,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -108,7 +110,6 @@ static void test_request(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test(config, "filter_causes_close", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -123,6 +124,7 @@ static void test_request(grpc_end2end_test_config config) { grpc_call_error error; grpc_slice details; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/filter_latency.c b/test/core/end2end/tests/filter_latency.c index 2428c92a42b..d38ce22e09b 100644 --- a/test/core/end2end/tests/filter_latency.c +++ b/test/core/end2end/tests/filter_latency.c @@ -69,16 +69,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -115,7 +117,6 @@ static void test_request(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test(config, "filter_latency", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -137,6 +138,7 @@ static void test_request(grpc_end2end_test_config config) { gpr_mu_unlock(&g_mu); const gpr_timespec start_time = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/graceful_server_shutdown.c b/test/core/end2end/tests/graceful_server_shutdown.c index a3ad260cc22..182bb207805 100644 --- a/test/core/end2end/tests/graceful_server_shutdown.c +++ b/test/core/end2end/tests/graceful_server_shutdown.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -95,7 +97,6 @@ static void test_early_server_shutdown_finishes_inflight_calls( grpc_end2end_test_config config) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = n_seconds_time(10); grpc_end2end_test_fixture f = begin_test( config, "test_early_server_shutdown_finishes_inflight_calls", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -110,6 +111,7 @@ static void test_early_server_shutdown_finishes_inflight_calls( grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = n_seconds_from_now(10); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/high_initial_seqno.c b/test/core/end2end/tests/high_initial_seqno.c index cca8532b0e0..f534a19c872 100644 --- a/test/core/end2end/tests/high_initial_seqno.c +++ b/test/core/end2end/tests/high_initial_seqno.c @@ -61,16 +61,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -103,7 +105,6 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -116,6 +117,7 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/hpack_size.c b/test/core/end2end/tests/hpack_size.c index 7601722deea..17ef204ce94 100644 --- a/test/core/end2end/tests/hpack_size.c +++ b/test/core/end2end/tests/hpack_size.c @@ -201,16 +201,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -243,7 +245,6 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f, size_t index) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -268,6 +269,7 @@ static void simple_request_body(grpc_end2end_test_config config, extra_metadata[2].value = grpc_slice_from_static_string(dragons[index % GPR_ARRAY_SIZE(dragons)]); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/idempotent_request.c b/test/core/end2end/tests/idempotent_request.c index cef2e100be9..f80654db1a2 100644 --- a/test/core/end2end/tests/idempotent_request.c +++ b/test/core/end2end/tests/idempotent_request.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -101,7 +103,6 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -115,6 +116,7 @@ static void simple_request_body(grpc_end2end_test_config config, int was_cancelled = 2; char *peer; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/invoke_large_request.c b/test/core/end2end/tests/invoke_large_request.c index d799bd8ccf9..f4b93d9873f 100644 --- a/test/core/end2end/tests/invoke_large_request.c +++ b/test/core/end2end/tests/invoke_large_request.c @@ -58,14 +58,14 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, n_seconds_time(5), NULL); + ev = grpc_completion_queue_next(cq, n_seconds_from_now(5), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -128,7 +128,6 @@ static void test_invoke_large_request(grpc_end2end_test_config config, grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_byte_buffer *response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1); - gpr_timespec deadline = n_seconds_time(30); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -143,6 +142,7 @@ static void test_invoke_large_request(grpc_end2end_test_config config, grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = n_seconds_from_now(30); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/keepalive_timeout.c b/test/core/end2end/tests/keepalive_timeout.c index 44b6e12abc7..b2252dc802e 100644 --- a/test/core/end2end/tests/keepalive_timeout.c +++ b/test/core/end2end/tests/keepalive_timeout.c @@ -61,25 +61,27 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } static void shutdown_server(grpc_end2end_test_fixture *f) { if (!f->server) return; grpc_server_shutdown_and_notify(f->server, f->cq, tag(1000)); - GPR_ASSERT( - grpc_completion_queue_pluck(f->cq, tag(1000), five_seconds_time(), NULL) - .type == GRPC_OP_COMPLETE); + GPR_ASSERT(grpc_completion_queue_pluck(f->cq, tag(1000), + five_seconds_from_now(), NULL) + .type == GRPC_OP_COMPLETE); grpc_server_destroy(f->server); f->server = NULL; } @@ -108,7 +110,6 @@ static void test_keepalive_timeout(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_arg keepalive_args[] = {{.type = GRPC_ARG_INTEGER, .key = GRPC_ARG_CLIENT_KEEPALIVE_TIME_S, @@ -140,6 +141,7 @@ static void test_keepalive_timeout(grpc_end2end_test_config config) { /* Disable ping ack to trigger the keepalive timeout */ grpc_set_disable_ping_ack(true); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/large_metadata.c b/test/core/end2end/tests/large_metadata.c index ac4c0e7f3b4..a670e411e9c 100644 --- a/test/core/end2end/tests/large_metadata.c +++ b/test/core/end2end/tests/large_metadata.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -103,7 +105,6 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_metadata meta; const size_t large_size = 64 * 1024; grpc_arg arg = {GRPC_ARG_INTEGER, @@ -125,6 +126,7 @@ static void test_request_with_large_metadata(grpc_end2end_test_config config) { grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/load_reporting_hook.c b/test/core/end2end/tests/load_reporting_hook.c index d1ee26fe502..b05943692c3 100644 --- a/test/core/end2end/tests/load_reporting_hook.c +++ b/test/core/end2end/tests/load_reporting_hook.c @@ -84,16 +84,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -136,7 +138,6 @@ static void request_response_with_payload( grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_byte_buffer *response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -151,6 +152,7 @@ static void request_response_with_payload( grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string(method_name), diff --git a/test/core/end2end/tests/max_concurrent_streams.c b/test/core/end2end/tests/max_concurrent_streams.c index e81a6289443..2360138ede3 100644 --- a/test/core/end2end/tests/max_concurrent_streams.c +++ b/test/core/end2end/tests/max_concurrent_streams.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -99,7 +101,6 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -112,6 +113,7 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), @@ -257,7 +259,7 @@ static void test_max_concurrent_streams(grpc_end2end_test_config config) { /* start two requests - ensuring that the second is not accepted until the first completes */ - deadline = n_seconds_time(1000); + deadline = n_seconds_from_now(1000); c1 = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/alpha"), @@ -502,13 +504,13 @@ static void test_max_concurrent_streams_with_timeout_on_first( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/alpha"), get_host_override_slice("foo.test.google.fr:1234", config), - n_seconds_time(3), NULL); + n_seconds_from_now(3), NULL); GPR_ASSERT(c1); c2 = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/beta"), get_host_override_slice("foo.test.google.fr:1234", config), - n_seconds_time(1000), NULL); + n_seconds_from_now(1000), NULL); GPR_ASSERT(c2); GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call( @@ -697,13 +699,13 @@ static void test_max_concurrent_streams_with_timeout_on_second( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/alpha"), get_host_override_slice("foo.test.google.fr:1234", config), - n_seconds_time(1000), NULL); + n_seconds_from_now(1000), NULL); GPR_ASSERT(c1); c2 = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/beta"), get_host_override_slice("foo.test.google.fr:1234", config), - n_seconds_time(3), NULL); + n_seconds_from_now(3), NULL); GPR_ASSERT(c2); GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call( diff --git a/test/core/end2end/tests/max_message_length.c b/test/core/end2end/tests/max_message_length.c index b15d30f58c4..ab58d9f9a62 100644 --- a/test/core/end2end/tests/max_message_length.c +++ b/test/core/end2end/tests/max_message_length.c @@ -66,16 +66,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } diff --git a/test/core/end2end/tests/negative_deadline.c b/test/core/end2end/tests/negative_deadline.c index 0b61efbac97..e7448bd6bc8 100644 --- a/test/core/end2end/tests/negative_deadline.c +++ b/test/core/end2end/tests/negative_deadline.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -100,7 +102,6 @@ static void end_test(grpc_end2end_test_fixture *f) { static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f, size_t num_ops) { grpc_call *c; - gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_REALTIME); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -112,6 +113,7 @@ static void simple_request_body(grpc_end2end_test_config config, gpr_log(GPR_DEBUG, "test with %" PRIuPTR " ops", num_ops); + gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_REALTIME); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/network_status_change.c b/test/core/end2end/tests/network_status_change.c index d7a4106459d..1a5f2b25cc2 100644 --- a/test/core/end2end/tests/network_status_change.c +++ b/test/core/end2end/tests/network_status_change.c @@ -60,16 +60,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(500); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(500); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -106,7 +108,6 @@ static void test_invoke_network_status_change(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test(config, "test_invoke_request_with_payload", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -122,6 +123,7 @@ static void test_invoke_network_status_change(grpc_end2end_test_config config) { grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/no_logging.c b/test/core/end2end/tests/no_logging.c index 56e48a88a87..683d6465db9 100644 --- a/test/core/end2end/tests/no_logging.c +++ b/test/core/end2end/tests/no_logging.c @@ -87,16 +87,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -129,7 +131,6 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -143,6 +144,7 @@ static void simple_request_body(grpc_end2end_test_config config, int was_cancelled = 2; char *peer; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/no_op.c b/test/core/end2end/tests/no_op.c index 62fc728c3eb..698d8fbc607 100644 --- a/test/core/end2end/tests/no_op.c +++ b/test/core/end2end/tests/no_op.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } diff --git a/test/core/end2end/tests/payload.c b/test/core/end2end/tests/payload.c index b04ee5705c6..5751b801b5d 100644 --- a/test/core/end2end/tests/payload.c +++ b/test/core/end2end/tests/payload.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -126,7 +128,6 @@ static void request_response_with_payload(grpc_end2end_test_config config, grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_byte_buffer *response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1); - gpr_timespec deadline = n_seconds_time(60); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -141,6 +142,7 @@ static void request_response_with_payload(grpc_end2end_test_config config, grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = n_seconds_from_now(60); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/ping_pong_streaming.c b/test/core/end2end/tests/ping_pong_streaming.c index 848f76018d1..cd81ce24feb 100644 --- a/test/core/end2end/tests/ping_pong_streaming.c +++ b/test/core/end2end/tests/ping_pong_streaming.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -102,7 +104,6 @@ static void test_pingpong_streaming(grpc_end2end_test_config config, begin_test(config, "test_pingpong_streaming", NULL, NULL); grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -124,6 +125,7 @@ static void test_pingpong_streaming(grpc_end2end_test_config config, grpc_slice response_payload_slice = grpc_slice_from_copied_string("hello you"); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/registered_call.c b/test/core/end2end/tests/registered_call.c index 9c8ce89c838..242add37cb8 100644 --- a/test/core/end2end/tests/registered_call.c +++ b/test/core/end2end/tests/registered_call.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -101,7 +103,6 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f, void *rc) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -114,6 +115,7 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_registered_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, rc, deadline, NULL); GPR_ASSERT(c); diff --git a/test/core/end2end/tests/request_with_flags.c b/test/core/end2end/tests/request_with_flags.c index 329359e08b8..10ecf5b1af5 100644 --- a/test/core/end2end/tests/request_with_flags.c +++ b/test/core/end2end/tests/request_with_flags.c @@ -58,16 +58,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -104,7 +106,6 @@ static void test_invoke_request_with_flags( grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test(config, "test_invoke_request_with_flags", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -120,6 +121,7 @@ static void test_invoke_request_with_flags( grpc_slice details; grpc_call_error expectation; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/request_with_payload.c b/test/core/end2end/tests/request_with_payload.c index f71f92bbb84..4eb3399bb95 100644 --- a/test/core/end2end/tests/request_with_payload.c +++ b/test/core/end2end/tests/request_with_payload.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -103,7 +105,6 @@ static void test_invoke_request_with_payload(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test(config, "test_invoke_request_with_payload", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -119,6 +120,7 @@ static void test_invoke_request_with_payload(grpc_end2end_test_config config) { grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/resource_quota_server.c b/test/core/end2end/tests/resource_quota_server.c index db26b4480e0..4d3ce1c937c 100644 --- a/test/core/end2end/tests/resource_quota_server.c +++ b/test/core/end2end/tests/resource_quota_server.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -195,7 +197,7 @@ void resource_quota_server(grpc_end2end_test_config config) { f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), get_host_override_slice("foo.test.google.fr", config), - n_seconds_time(60), NULL); + n_seconds_from_now(60), NULL); memset(ops, 0, sizeof(ops)); op = ops; @@ -237,7 +239,8 @@ void resource_quota_server(grpc_end2end_test_config config) { while (pending_client_calls + pending_server_recv_calls + pending_server_end_calls > 0) { - grpc_event ev = grpc_completion_queue_next(f.cq, n_seconds_time(60), NULL); + grpc_event ev = + grpc_completion_queue_next(f.cq, n_seconds_from_now(60), NULL); GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); int ev_tag = (int)(intptr_t)ev.tag; diff --git a/test/core/end2end/tests/server_finishes_request.c b/test/core/end2end/tests/server_finishes_request.c index b42d17002e8..88bc90e7788 100644 --- a/test/core/end2end/tests/server_finishes_request.c +++ b/test/core/end2end/tests/server_finishes_request.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -101,7 +103,6 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -114,6 +115,7 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/shutdown_finishes_calls.c b/test/core/end2end/tests/shutdown_finishes_calls.c index c019682ea66..6f9421d19a7 100644 --- a/test/core/end2end/tests/shutdown_finishes_calls.c +++ b/test/core/end2end/tests/shutdown_finishes_calls.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -88,7 +90,6 @@ static void test_early_server_shutdown_finishes_inflight_calls( grpc_end2end_test_config config) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test( config, "test_early_server_shutdown_finishes_inflight_calls", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -103,6 +104,7 @@ static void test_early_server_shutdown_finishes_inflight_calls( grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/shutdown_finishes_tags.c b/test/core/end2end/tests/shutdown_finishes_tags.c index 5540d2aab9c..51a242337f1 100644 --- a/test/core/end2end/tests/shutdown_finishes_tags.c +++ b/test/core/end2end/tests/shutdown_finishes_tags.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } diff --git a/test/core/end2end/tests/simple_cacheable_request.c b/test/core/end2end/tests/simple_cacheable_request.c index 4eef02e9eed..4f1013a0300 100644 --- a/test/core/end2end/tests/simple_cacheable_request.c +++ b/test/core/end2end/tests/simple_cacheable_request.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -110,7 +112,6 @@ static void test_cacheable_request_response_with_metadata_and_payload( grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_byte_buffer *response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_metadata meta_c[2] = {{grpc_slice_from_static_string("key1"), grpc_slice_from_static_string("val1"), 0, @@ -144,6 +145,7 @@ static void test_cacheable_request_response_with_metadata_and_payload( grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/simple_delayed_request.c b/test/core/end2end/tests/simple_delayed_request.c index e3b6aee783e..978dff0dc0e 100644 --- a/test/core/end2end/tests/simple_delayed_request.c +++ b/test/core/end2end/tests/simple_delayed_request.c @@ -45,16 +45,18 @@ static void *tag(intptr_t t) { return (void *)t; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -90,7 +92,6 @@ static void simple_delayed_request_body(grpc_end2end_test_config config, long delay_us) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f->cq); grpc_op ops[6]; grpc_op *op; @@ -106,6 +107,7 @@ static void simple_delayed_request_body(grpc_end2end_test_config config, config.init_client(f, client_args); config.init_server(f, server_args); + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f->client, NULL, GRPC_PROPAGATE_DEFAULTS, f->cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/simple_metadata.c b/test/core/end2end/tests/simple_metadata.c index 7ab5563cfa9..83d1037e4bc 100644 --- a/test/core/end2end/tests/simple_metadata.c +++ b/test/core/end2end/tests/simple_metadata.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -108,7 +110,6 @@ static void test_request_response_with_metadata_and_payload( grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_byte_buffer *response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_metadata meta_c[2] = {{grpc_slice_from_static_string("key1"), grpc_slice_from_static_string("val1"), 0, @@ -141,6 +142,7 @@ static void test_request_response_with_metadata_and_payload( grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/simple_request.c b/test/core/end2end/tests/simple_request.c index af5d74959e3..31bfef4cc24 100644 --- a/test/core/end2end/tests/simple_request.c +++ b/test/core/end2end/tests/simple_request.c @@ -59,16 +59,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -101,7 +103,6 @@ static void simple_request_body(grpc_end2end_test_config config, grpc_end2end_test_fixture f) { grpc_call *c; grpc_call *s; - gpr_timespec deadline = five_seconds_time(); cq_verifier *cqv = cq_verifier_create(f.cq); grpc_op ops[6]; grpc_op *op; @@ -115,6 +116,7 @@ static void simple_request_body(grpc_end2end_test_config config, int was_cancelled = 2; char *peer; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/streaming_error_response.c b/test/core/end2end/tests/streaming_error_response.c index 2b9c404b15a..c652d9469d5 100644 --- a/test/core/end2end/tests/streaming_error_response.c +++ b/test/core/end2end/tests/streaming_error_response.c @@ -62,16 +62,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -110,7 +112,6 @@ static void test(grpc_end2end_test_config config, bool request_status_early) { grpc_slice response_payload2_slice = grpc_slice_from_copied_string("world"); grpc_byte_buffer *response_payload2 = grpc_raw_byte_buffer_create(&response_payload2_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test(config, "streaming_error_response", NULL, NULL, request_status_early); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -127,6 +128,7 @@ static void test(grpc_end2end_test_config config, bool request_status_early) { grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/trailing_metadata.c b/test/core/end2end/tests/trailing_metadata.c index dbbda505bc4..3cbffdfe754 100644 --- a/test/core/end2end/tests/trailing_metadata.c +++ b/test/core/end2end/tests/trailing_metadata.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -108,7 +110,6 @@ static void test_request_response_with_metadata_and_payload( grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_byte_buffer *response_payload = grpc_raw_byte_buffer_create(&response_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_metadata meta_c[2] = {{grpc_slice_from_static_string("key1"), grpc_slice_from_static_string("val1"), 0, @@ -149,6 +150,7 @@ static void test_request_response_with_metadata_and_payload( grpc_slice details; int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/write_buffering.c b/test/core/end2end/tests/write_buffering.c index abf90ca6e06..2d4142b97d3 100644 --- a/test/core/end2end/tests/write_buffering.c +++ b/test/core/end2end/tests/write_buffering.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -106,7 +108,6 @@ static void test_invoke_request_with_payload(grpc_end2end_test_config config) { grpc_slice request_payload_slice2 = grpc_slice_from_copied_string("abc123"); grpc_byte_buffer *request_payload2 = grpc_raw_byte_buffer_create(&request_payload_slice2, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test(config, "test_invoke_request_with_payload", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -123,6 +124,7 @@ static void test_invoke_request_with_payload(grpc_end2end_test_config config) { grpc_slice details = grpc_empty_slice(); int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), diff --git a/test/core/end2end/tests/write_buffering_at_end.c b/test/core/end2end/tests/write_buffering_at_end.c index 8c02b425bae..e61343e6842 100644 --- a/test/core/end2end/tests/write_buffering_at_end.c +++ b/test/core/end2end/tests/write_buffering_at_end.c @@ -57,16 +57,18 @@ static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config, return f; } -static gpr_timespec n_seconds_time(int n) { +static gpr_timespec n_seconds_from_now(int n) { return grpc_timeout_seconds_to_deadline(n); } -static gpr_timespec five_seconds_time(void) { return n_seconds_time(5); } +static gpr_timespec five_seconds_from_now(void) { + return n_seconds_from_now(5); +} static void drain_cq(grpc_completion_queue *cq) { grpc_event ev; do { - ev = grpc_completion_queue_next(cq, five_seconds_time(), NULL); + ev = grpc_completion_queue_next(cq, five_seconds_from_now(), NULL); } while (ev.type != GRPC_QUEUE_SHUTDOWN); } @@ -103,7 +105,6 @@ static void test_invoke_request_with_payload(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); grpc_end2end_test_fixture f = begin_test(config, "test_invoke_request_with_payload", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -120,6 +121,7 @@ static void test_invoke_request_with_payload(grpc_end2end_test_config config) { grpc_slice details = grpc_empty_slice(); int was_cancelled = 2; + gpr_timespec deadline = five_seconds_from_now(); c = grpc_channel_create_call( f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, grpc_slice_from_static_string("/foo"), From e3b5921559bb26fa49c10d6019cbb97634500059 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 4 Apr 2017 09:28:54 -0700 Subject: [PATCH 54/90] Potential msan fix --- include/grpc++/impl/codegen/call.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index a3f2be6bb1e..e0095b0434c 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -528,6 +528,7 @@ class CallOpClientRecvStatus { void ClientRecvStatus(ClientContext* context, Status* status) { metadata_map_ = &context->trailing_metadata_; recv_status_ = status; + status_details_ = grpc_empty_slice(); } protected: From be094fc5e6a82b9e5904176a70e74bb66fbb32f7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 4 Apr 2017 09:31:28 -0700 Subject: [PATCH 55/90] Fix compile --- include/grpc++/impl/codegen/call.h | 2 +- include/grpc++/impl/codegen/core_codegen.h | 1 + include/grpc++/impl/codegen/core_codegen_interface.h | 1 + src/cpp/common/core_codegen.cc | 2 ++ 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index e0095b0434c..be6857c4829 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -528,7 +528,7 @@ class CallOpClientRecvStatus { void ClientRecvStatus(ClientContext* context, Status* status) { metadata_map_ = &context->trailing_metadata_; recv_status_ = status; - status_details_ = grpc_empty_slice(); + status_details_ = g_core_codegen_interface->grpc_empty_slice(); } protected: diff --git a/include/grpc++/impl/codegen/core_codegen.h b/include/grpc++/impl/codegen/core_codegen.h index 754bf14b259..6bf95129b81 100644 --- a/include/grpc++/impl/codegen/core_codegen.h +++ b/include/grpc++/impl/codegen/core_codegen.h @@ -76,6 +76,7 @@ class CoreCodegen : public CoreCodegenInterface { grpc_byte_buffer* grpc_raw_byte_buffer_create(grpc_slice* slice, size_t nslices) override; + grpc_slice grpc_empty_slice() override; grpc_slice grpc_slice_malloc(size_t length) override; void grpc_slice_unref(grpc_slice slice) override; grpc_slice grpc_slice_split_tail(grpc_slice* s, size_t split) override; diff --git a/include/grpc++/impl/codegen/core_codegen_interface.h b/include/grpc++/impl/codegen/core_codegen_interface.h index 45ea0403031..e111d59364b 100644 --- a/include/grpc++/impl/codegen/core_codegen_interface.h +++ b/include/grpc++/impl/codegen/core_codegen_interface.h @@ -94,6 +94,7 @@ class CoreCodegenInterface { virtual grpc_byte_buffer* grpc_raw_byte_buffer_create(grpc_slice* slice, size_t nslices) = 0; + virtual grpc_slice grpc_empty_slice() = 0; virtual grpc_slice grpc_slice_malloc(size_t length) = 0; virtual void grpc_slice_unref(grpc_slice slice) = 0; virtual grpc_slice grpc_slice_split_tail(grpc_slice* s, size_t split) = 0; diff --git a/src/cpp/common/core_codegen.cc b/src/cpp/common/core_codegen.cc index 36e4c893540..43dffe7a2a9 100644 --- a/src/cpp/common/core_codegen.cc +++ b/src/cpp/common/core_codegen.cc @@ -111,6 +111,8 @@ grpc_byte_buffer* CoreCodegen::grpc_raw_byte_buffer_create(grpc_slice* slice, return ::grpc_raw_byte_buffer_create(slice, nslices); } +grpc_slice CoreCodegen::grpc_empty_slice() { return ::grpc_empty_slice(); } + grpc_slice CoreCodegen::grpc_slice_malloc(size_t length) { return ::grpc_slice_malloc(length); } From 61d02025b55030cdae26a72b001748af2a0d1424 Mon Sep 17 00:00:00 2001 From: yang-g Date: Tue, 4 Apr 2017 10:04:18 -0700 Subject: [PATCH 56/90] Unref the tail, not the original slice --- src/core/lib/security/transport/security_handshaker.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/core/lib/security/transport/security_handshaker.c b/src/core/lib/security/transport/security_handshaker.c index 2f393276707..509b4b556d6 100644 --- a/src/core/lib/security/transport/security_handshaker.c +++ b/src/core/lib/security/transport/security_handshaker.c @@ -287,12 +287,11 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx, if (num_left_overs > 0) { /* Put the leftovers in our buffer (ownership transfered). */ if (has_left_overs_in_current_slice) { - grpc_slice_buffer_add( - &h->left_overs, - grpc_slice_split_tail(&h->args->read_buffer->slices[i], - consumed_slice_size)); + grpc_slice tail = grpc_slice_split_tail(&h->args->read_buffer->slices[i], + consumed_slice_size); + grpc_slice_buffer_add(&h->left_overs, tail); /* split_tail above increments refcount. */ - grpc_slice_unref_internal(exec_ctx, h->args->read_buffer->slices[i]); + grpc_slice_unref_internal(exec_ctx, tail); } grpc_slice_buffer_addn( &h->left_overs, &h->args->read_buffer->slices[i + 1], From 5ce09f08d2ba9eb8b7e75273bcd92c0d7af1b89e Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 4 Apr 2017 10:21:21 -0700 Subject: [PATCH 57/90] Disable CronetUnitTests for possible Jenkins flakiness --- src/objective-c/tests/run_tests.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/objective-c/tests/run_tests.sh b/src/objective-c/tests/run_tests.sh index bd7c2945a27..0e82bcaa44a 100755 --- a/src/objective-c/tests/run_tests.sh +++ b/src/objective-c/tests/run_tests.sh @@ -68,12 +68,16 @@ xcodebuild \ -destination name="iPhone 6" \ test | xcpretty -echo "TIME: $(date)" -xcodebuild \ - -workspace Tests.xcworkspace \ - -scheme CronetUnitTests \ - -destination name="iPhone 6" \ - test | xcpretty +# Temporarily disabled for (possible) flakiness on Jenkins. +# Fix or reenable after confirmation/disconfirmation that it is the source of +# Jenkins problem. + +# echo "TIME: $(date)" +# xcodebuild \ +# -workspace Tests.xcworkspace \ +# -scheme CronetUnitTests \ +# -destination name="iPhone 6" \ +# test | xcpretty echo "TIME: $(date)" xcodebuild \ From b0c2225336d1a71ebb32f850cbec968ef9c96820 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 4 Apr 2017 11:08:19 -0700 Subject: [PATCH 58/90] generate projects --- tools/run_tests/generated/tests.json | 194 +++++++++++++-------------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index d40d5717f41..88bac795e49 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -24359,29 +24359,6 @@ "posix" ] }, - { - "args": [ - "bad_ping" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_uds_test", - "platforms": [ - "linux", - "mac", - "posix" - ] - }, { "args": [ "binary_metadata" @@ -25417,19 +25394,19 @@ "bad_ping" ], "ci_platforms": [ - "windows", "linux", "mac", "posix" ], "cpu_cost": 1.0, "exclude_configs": [], - "exclude_iomgrs": [], + "exclude_iomgrs": [ + "uv" + ], "flaky": false, "language": "c", - "name": "h2_census_nosec_test", + "name": "h2_uds_test", "platforms": [ - "windows", "linux", "mac", "posix" @@ -26623,7 +26600,7 @@ "exclude_iomgrs": [], "flaky": false, "language": "c", - "name": "h2_compress_nosec_test", + "name": "h2_census_nosec_test", "platforms": [ "windows", "linux", @@ -27808,6 +27785,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -28985,29 +28985,6 @@ "posix" ] }, - { - "args": [ - "bad_ping" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "h2_full_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, { "args": [ "binary_metadata" @@ -30048,18 +30025,22 @@ "bad_ping" ], "ci_platforms": [ - "linux" + "windows", + "linux", + "mac", + "posix" ], "cpu_cost": 1.0, "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], + "exclude_iomgrs": [], "flaky": false, "language": "c", - "name": "h2_full+pipe_nosec_test", + "name": "h2_full_nosec_test", "platforms": [ - "linux" + "windows", + "linux", + "mac", + "posix" ] }, { @@ -31236,22 +31217,18 @@ "bad_ping" ], "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" + "linux" ], "cpu_cost": 1.0, "exclude_configs": [], - "exclude_iomgrs": [], + "exclude_iomgrs": [ + "uv" + ], "flaky": false, "language": "c", - "name": "h2_full+trace_nosec_test", + "name": "h2_full+pipe_nosec_test", "platforms": [ - "windows", - "linux", - "mac", - "posix" + "linux" ] }, { @@ -32238,16 +32215,15 @@ "ci_platforms": [ "windows", "linux", + "mac", "posix" ], "cpu_cost": 1.0, "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], + "exclude_iomgrs": [], "flaky": false, "language": "c", - "name": "h2_http_proxy_nosec_test", + "name": "h2_full+trace_nosec_test", "platforms": [ "windows", "linux", @@ -33395,15 +33371,16 @@ "ci_platforms": [ "windows", "linux", - "mac", "posix" ], "cpu_cost": 1.0, "exclude_configs": [], - "exclude_iomgrs": [], + "exclude_iomgrs": [ + "uv" + ], "flaky": false, "language": "c", - "name": "h2_load_reporting_nosec_test", + "name": "h2_http_proxy_nosec_test", "platforms": [ "windows", "linux", @@ -34633,6 +34610,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "language": "c", + "name": "h2_load_reporting_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" @@ -38912,29 +38912,6 @@ "posix" ] }, - { - "args": [ - "bad_ping" - ], - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "language": "c", - "name": "h2_uds_nosec_test", - "platforms": [ - "linux", - "mac", - "posix" - ] - }, { "args": [ "binary_metadata" @@ -40073,6 +40050,29 @@ "posix" ] }, + { + "args": [ + "bad_ping" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "language": "c", + "name": "h2_uds_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "binary_metadata" From 7477a14e3caa36d77c3c19789e63bb1676d65054 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 4 Apr 2017 13:26:44 -0700 Subject: [PATCH 59/90] Remove invalid gpr_free() of arena-allocated memory. --- src/core/ext/filters/client_channel/subchannel.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c index 8086cdb87e0..29a1a095553 100644 --- a/src/core/ext/filters/client_channel/subchannel.c +++ b/src/core/ext/filters/client_channel/subchannel.c @@ -782,8 +782,6 @@ grpc_error *grpc_connected_subchannel_create_call( if (error != GRPC_ERROR_NONE) { const char *error_string = grpc_error_string(error); gpr_log(GPR_ERROR, "error: %s", error_string); - - gpr_free(*call); return error; } GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call"); From ca13603302e7e256ec0e950ab7b133782920facc Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 4 Apr 2017 13:53:29 -0700 Subject: [PATCH 60/90] Do not retry when we fail to create a subchannel call. --- .../ext/filters/client_channel/client_channel.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index 93ad53aab93..04b0ffdaf3d 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -1154,14 +1154,17 @@ static void start_transport_stream_op_batch_locked_inner( exec_ctx, calld->connected_subchannel, &call_args, &subchannel_call); if (error != GRPC_ERROR_NONE) { subchannel_call = CANCELLED_CALL; + gpr_atm_rel_store(&calld->subchannel_call, + (gpr_atm)(uintptr_t)subchannel_call); fail_locked(exec_ctx, calld, GRPC_ERROR_REF(error)); grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error); + } else { + gpr_atm_rel_store(&calld->subchannel_call, + (gpr_atm)(uintptr_t)subchannel_call); + retry_waiting_locked(exec_ctx, calld); + /* recurse to retry */ + start_transport_stream_op_batch_locked_inner(exec_ctx, op, elem); } - gpr_atm_rel_store(&calld->subchannel_call, - (gpr_atm)(uintptr_t)subchannel_call); - retry_waiting_locked(exec_ctx, calld); - /* recurse to retry */ - start_transport_stream_op_batch_locked_inner(exec_ctx, op, elem); /* early out */ return; } From a2ca7247ae7345dce987a3542a816f4e101a9345 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Thu, 30 Mar 2017 15:31:29 -0700 Subject: [PATCH 61/90] Add max_connection_age jitter --- src/core/ext/filters/max_age/max_age_filter.c | 24 ++++++++++++++++--- test/core/end2end/tests/max_connection_age.c | 5 ++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/core/ext/filters/max_age/max_age_filter.c b/src/core/ext/filters/max_age/max_age_filter.c index b03cb0ba0a9..be5350bce5e 100644 --- a/src/core/ext/filters/max_age/max_age_filter.c +++ b/src/core/ext/filters/max_age/max_age_filter.c @@ -46,6 +46,7 @@ #define DEFAULT_MAX_CONNECTION_AGE_MS INT_MAX #define DEFAULT_MAX_CONNECTION_AGE_GRACE_MS INT_MAX #define DEFAULT_MAX_CONNECTION_IDLE_MS INT_MAX +#define MAX_CONNECTION_AGE_JITTER 0.1 typedef struct channel_data { /* We take a reference to the channel stack for the timer callback */ @@ -254,6 +255,19 @@ static void channel_connectivity_changed(grpc_exec_ctx* exec_ctx, void* arg, } } +/* A random jitter of +/-10% will be added to MAX_CONNECTION_AGE to spread out + connection storms. Note that the MAX_CONNECTION_AGE option without jitter + would not create connection storms by itself, but if there happened to be a + connection storm it could cause it to repeat at a fixed period. */ +static int add_random_max_connection_age_jitter(int value) { + /* generate a random number between 1 - MAX_CONNECTION_AGE_JITTER and + 1 + MAX_CONNECTION_AGE_JITTER */ + double multiplier = rand() * MAX_CONNECTION_AGE_JITTER * 2.0 / RAND_MAX + + 1.0 - MAX_CONNECTION_AGE_JITTER; + double result = multiplier * value; + return result > INT_MAX ? INT_MAX : (int)result; +} + /* Constructor for call_data. */ static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx, grpc_call_element* elem, @@ -283,7 +297,9 @@ static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx, chand->max_connection_age = DEFAULT_MAX_CONNECTION_AGE_MS == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(DEFAULT_MAX_CONNECTION_AGE_MS, GPR_TIMESPAN); + : gpr_time_from_millis(add_random_max_connection_age_jitter( + DEFAULT_MAX_CONNECTION_AGE_MS), + GPR_TIMESPAN); chand->max_connection_age_grace = DEFAULT_MAX_CONNECTION_AGE_GRACE_MS == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) @@ -300,8 +316,10 @@ static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx, &args->channel_args->args[i], (grpc_integer_options){DEFAULT_MAX_CONNECTION_AGE_MS, 1, INT_MAX}); chand->max_connection_age = - value == INT_MAX ? gpr_inf_future(GPR_TIMESPAN) - : gpr_time_from_millis(value, GPR_TIMESPAN); + value == INT_MAX + ? gpr_inf_future(GPR_TIMESPAN) + : gpr_time_from_millis( + add_random_max_connection_age_jitter(value), GPR_TIMESPAN); } else if (0 == strcmp(args->channel_args->args[i].key, GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS)) { const int value = grpc_channel_arg_get_integer( diff --git a/test/core/end2end/tests/max_connection_age.c b/test/core/end2end/tests/max_connection_age.c index 59bfdbabb9b..04bdb39445f 100644 --- a/test/core/end2end/tests/max_connection_age.c +++ b/test/core/end2end/tests/max_connection_age.c @@ -47,6 +47,7 @@ #define MAX_CONNECTION_AGE_GRACE_MS 1000 #define MAX_CONNECTION_IDLE_MS 9999 +#define MAX_CONNECTION_AGE_JITTER_MULTIPLIER 1.1 #define CALL_DEADLINE_S 10 /* The amount of time we wait for the connection to time out, but after it the connection should not use up its grace period. It should be a number between @@ -169,8 +170,8 @@ static void test_max_age_forcibly_close(grpc_end2end_test_config config) { cq_verify(cqv); gpr_timespec expect_shutdown_time = grpc_timeout_milliseconds_to_deadline( - MAX_CONNECTION_AGE_MS + MAX_CONNECTION_AGE_GRACE_MS + - IMMEDIATE_SHUTDOWN_GRACE_TIME_MS); + (int)(MAX_CONNECTION_AGE_MS * MAX_CONNECTION_AGE_JITTER_MULTIPLIER) + + MAX_CONNECTION_AGE_GRACE_MS + IMMEDIATE_SHUTDOWN_GRACE_TIME_MS); /* Wait for the channel to reach its max age */ cq_verify_empty_timeout(cqv, CQ_MAX_CONNECTION_AGE_WAIT_TIME_S); From 0b3d1360c2d56e32ec081304c6f2c65eb3ab1766 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 4 Apr 2017 13:55:33 -0700 Subject: [PATCH 62/90] Fix float comparison --- src/core/ext/filters/max_age/max_age_filter.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/ext/filters/max_age/max_age_filter.c b/src/core/ext/filters/max_age/max_age_filter.c index be5350bce5e..f858220c010 100644 --- a/src/core/ext/filters/max_age/max_age_filter.c +++ b/src/core/ext/filters/max_age/max_age_filter.c @@ -265,7 +265,9 @@ static int add_random_max_connection_age_jitter(int value) { double multiplier = rand() * MAX_CONNECTION_AGE_JITTER * 2.0 / RAND_MAX + 1.0 - MAX_CONNECTION_AGE_JITTER; double result = multiplier * value; - return result > INT_MAX ? INT_MAX : (int)result; + /* INT_MAX - 0.5 converts the value to float, so that result will not be + cast to int implicitly before the comparison. */ + return result > INT_MAX - 0.5 ? INT_MAX : (int)result; } /* Constructor for call_data. */ From 3c754e471619f20ceba807a35b40b2decbcb54b3 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 4 Apr 2017 13:49:08 -0700 Subject: [PATCH 63/90] Microbenchmark for measuring impact of multiple threads calling cq_next --- CMakeLists.txt | 46 ++++++ Makefile | 49 ++++++ build.yaml | 21 +++ src/core/lib/iomgr/ev_posix.c | 6 + src/core/lib/iomgr/ev_posix.h | 3 + test/cpp/microbenchmarks/BUILD | 23 ++- .../microbenchmarks/bm_cq_multiple_threads.cc | 146 ++++++++++++++++++ .../generated/sources_and_headers.json | 21 +++ tools/run_tests/generated/tests.json | 22 +++ 9 files changed, 333 insertions(+), 4 deletions(-) create mode 100644 test/cpp/microbenchmarks/bm_cq_multiple_threads.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index e3b2558ad29..d260a3797b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -626,6 +626,9 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_cq) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_dependencies(buildtests_cxx bm_cq_multiple_threads) +endif() +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx bm_error) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -8982,6 +8985,49 @@ endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) +add_executable(bm_cq_multiple_threads + test/cpp/microbenchmarks/bm_cq_multiple_threads.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(bm_cq_multiple_threads + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_BUILD_INCLUDE_DIR} + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(bm_cq_multiple_threads + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_benchmark + benchmark + grpc++_test_util + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + add_executable(bm_error test/cpp/microbenchmarks/bm_error.cc third_party/googletest/src/gtest-all.cc diff --git a/Makefile b/Makefile index 7499dbbf50d..0d52cb40052 100644 --- a/Makefile +++ b/Makefile @@ -1105,6 +1105,7 @@ bm_chttp2_hpack: $(BINDIR)/$(CONFIG)/bm_chttp2_hpack bm_chttp2_transport: $(BINDIR)/$(CONFIG)/bm_chttp2_transport bm_closure: $(BINDIR)/$(CONFIG)/bm_closure bm_cq: $(BINDIR)/$(CONFIG)/bm_cq +bm_cq_multiple_threads: $(BINDIR)/$(CONFIG)/bm_cq_multiple_threads bm_error: $(BINDIR)/$(CONFIG)/bm_error bm_fullstack_streaming_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump @@ -1530,6 +1531,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_chttp2_transport \ $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_cq \ + $(BINDIR)/$(CONFIG)/bm_cq_multiple_threads \ $(BINDIR)/$(CONFIG)/bm_error \ $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ @@ -1648,6 +1650,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/bm_chttp2_transport \ $(BINDIR)/$(CONFIG)/bm_closure \ $(BINDIR)/$(CONFIG)/bm_cq \ + $(BINDIR)/$(CONFIG)/bm_cq_multiple_threads \ $(BINDIR)/$(CONFIG)/bm_error \ $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong \ $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump \ @@ -1997,6 +2000,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/bm_closure || ( echo test bm_closure failed ; exit 1 ) $(E) "[RUN] Testing bm_cq" $(Q) $(BINDIR)/$(CONFIG)/bm_cq || ( echo test bm_cq failed ; exit 1 ) + $(E) "[RUN] Testing bm_cq_multiple_threads" + $(Q) $(BINDIR)/$(CONFIG)/bm_cq_multiple_threads || ( echo test bm_cq_multiple_threads failed ; exit 1 ) $(E) "[RUN] Testing bm_error" $(Q) $(BINDIR)/$(CONFIG)/bm_error || ( echo test bm_error failed ; exit 1 ) $(E) "[RUN] Testing bm_fullstack_streaming_ping_pong" @@ -13363,6 +13368,50 @@ endif endif +BM_CQ_MULTIPLE_THREADS_SRC = \ + test/cpp/microbenchmarks/bm_cq_multiple_threads.cc \ + +BM_CQ_MULTIPLE_THREADS_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_CQ_MULTIPLE_THREADS_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/bm_cq_multiple_threads: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/bm_cq_multiple_threads: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/bm_cq_multiple_threads: $(PROTOBUF_DEP) $(BM_CQ_MULTIPLE_THREADS_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(BM_CQ_MULTIPLE_THREADS_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_cq_multiple_threads + +endif + +endif + +$(BM_CQ_MULTIPLE_THREADS_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX +$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_cq_multiple_threads.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_bm_cq_multiple_threads: $(BM_CQ_MULTIPLE_THREADS_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(BM_CQ_MULTIPLE_THREADS_OBJS:.o=.dep) +endif +endif + + BM_ERROR_SRC = \ test/cpp/microbenchmarks/bm_error.cc \ diff --git a/build.yaml b/build.yaml index 8aff16854ce..c88b1161d2f 100644 --- a/build.yaml +++ b/build.yaml @@ -3205,6 +3205,27 @@ targets: - mac - linux - posix +- name: bm_cq_multiple_threads + build: test + language: c++ + src: + - test/cpp/microbenchmarks/bm_cq_multiple_threads.cc + deps: + - grpc_benchmark + - benchmark + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + args: + - --benchmark_min_time=0 + defaults: benchmark + platforms: + - mac + - linux + - posix - name: bm_error build: test language: c++ diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c index b5be5504b9d..13409a4de83 100644 --- a/src/core/lib/iomgr/ev_posix.c +++ b/src/core/lib/iomgr/ev_posix.c @@ -111,6 +111,12 @@ static void try_engine(const char *engine) { } } +/* This should be used for testing purposes ONLY */ +void grpc_set_event_engine_test_only( + const grpc_event_engine_vtable *ev_engine) { + g_event_engine = ev_engine; +} + /* Call this only after calling grpc_event_engine_init() */ const char *grpc_get_poll_strategy_name() { return g_poll_strategy_name; } diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index 1a9e5c115ae..becc4d359e5 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -183,4 +183,7 @@ void grpc_pollset_set_del_fd(grpc_exec_ctx *exec_ctx, typedef int (*grpc_poll_function_type)(struct pollfd *, nfds_t, int); extern grpc_poll_function_type grpc_poll_function; +/* This should be used for testing purposes ONLY */ +void grpc_set_event_engine_test_only(const grpc_event_engine_vtable *); + #endif /* GRPC_CORE_LIB_IOMGR_EV_POSIX_H */ diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index 38619666dcb..cae3fa1a146 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -32,16 +32,25 @@ licenses(["notice"]) # 3-clause BSD cc_test( name = "noop-benchmark", srcs = ["noop-benchmark.cc"], - deps = ["//external:benchmark"], linkopts = ["-pthread"], + deps = ["//external:benchmark"], ) cc_library( name = "helpers", srcs = ["helpers.cc"], - hdrs = ["helpers.h", "fullstack_fixtures.h", "fullstack_context_mutators.h"], - deps = ["//:grpc++", "//external:benchmark", "//test/core/util:grpc_test_util", "//src/proto/grpc/testing:echo_proto"], + hdrs = [ + "fullstack_context_mutators.h", + "fullstack_fixtures.h", + "helpers.h", + ], linkopts = ["-pthread"], + deps = [ + "//:grpc++", + "//external:benchmark", + "//src/proto/grpc/testing:echo_proto", + "//test/core/util:grpc_test_util", + ], ) cc_test( @@ -56,6 +65,12 @@ cc_test( deps = [":helpers"], ) +cc_test( + name = "bm_cq_multiple_threads", + srcs = ["bm_cq_multiple_threads.cc"], + deps = [":helpers"], +) + cc_test( name = "bm_error", srcs = ["bm_error.cc"], @@ -66,8 +81,8 @@ cc_test( name = "bm_fullstack_streaming_ping_pong", srcs = ["bm_fullstack_streaming_ping_pong.cc"], deps = [":helpers"], +) - ) cc_test( name = "bm_fullstack_streaming_pump", srcs = ["bm_fullstack_streaming_pump.cc"], diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc new file mode 100644 index 00000000000..f68baad2b01 --- /dev/null +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -0,0 +1,146 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include +#include + +#include +#include "test/cpp/microbenchmarks/helpers.h" + +extern "C" { +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/port.h" +#include "src/core/lib/surface/completion_queue.h" +} + +struct grpc_pollset { + gpr_mu mu; +}; + +namespace grpc { +namespace testing { + +static void* make_tag(int i) { return (void*)(intptr_t)i; } +static grpc_completion_queue* g_cq; +static grpc_event_engine_vtable g_vtable; + +static __thread int g_thread_idx; +static __thread grpc_cq_completion g_cq_completion; + +static void pollset_shutdown(grpc_exec_ctx* exec_ctx, grpc_pollset* ps, + grpc_closure* closure) { + grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_NONE); +} + +static void pollset_init(grpc_pollset* ps, gpr_mu** mu) { + gpr_mu_init(&ps->mu); + *mu = &ps->mu; +} + +static void pollset_destroy(grpc_pollset* ps) { gpr_mu_destroy(&ps->mu); } + +static grpc_error* pollset_kick(grpc_pollset* p, grpc_pollset_worker* worker) { + return GRPC_ERROR_NONE; +} + +/* Callback when the tag is dequeued from the completion queue. Does nothing */ +static void cq_done_cb(grpc_exec_ctx* exec_ctx, void* done_arg, + grpc_cq_completion* cq_completion) {} + +/* Queues a completion tag. ZERO polling overhead */ +static grpc_error* pollset_work(grpc_exec_ctx* exec_ctx, grpc_pollset* ps, + grpc_pollset_worker** worker, gpr_timespec now, + gpr_timespec deadline) { + gpr_mu_unlock(&ps->mu); + grpc_cq_end_op(exec_ctx, g_cq, make_tag(g_thread_idx), GRPC_ERROR_NONE, + cq_done_cb, NULL, &g_cq_completion); + grpc_exec_ctx_flush(exec_ctx); + gpr_mu_lock(&ps->mu); + return GRPC_ERROR_NONE; +} + +static void init_engine_vtable() { + memset(&g_vtable, 0, sizeof(g_vtable)); + + g_vtable.pollset_size = sizeof(grpc_pollset); + g_vtable.pollset_init = pollset_init; + g_vtable.pollset_shutdown = pollset_shutdown; + g_vtable.pollset_destroy = pollset_destroy; + g_vtable.pollset_work = pollset_work; + g_vtable.pollset_kick = pollset_kick; +} + +static void setup() { + grpc_init(); + init_engine_vtable(); + grpc_set_event_engine_test_only(&g_vtable); + + g_cq = grpc_completion_queue_create(NULL); +} + +static void BM_Cq_Throughput(benchmark::State& state) { + TrackCounters track_counters; + gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC); + + if (state.thread_index == 0) { + setup(); + } + + while (state.KeepRunning()) { + g_thread_idx = state.thread_index; + void* dummy_tag = make_tag(g_thread_idx); + grpc_cq_begin_op(g_cq, dummy_tag); + grpc_completion_queue_next(g_cq, deadline, NULL); + } + + if (state.thread_index == 0) { + grpc_completion_queue_shutdown(g_cq); + grpc_completion_queue_destroy(g_cq); + + state.SetItemsProcessed(state.iterations()); + } + + track_counters.Finish(state); +} + +BENCHMARK(BM_Cq_Throughput)->Threads(1); +BENCHMARK(BM_Cq_Throughput)->Threads(2); +BENCHMARK(BM_Cq_Throughput)->Threads(4); +BENCHMARK(BM_Cq_Throughput)->Threads(8); +BENCHMARK(BM_Cq_Throughput)->Threads(16); + +} // namespace testing +} // namespace grpc + +BENCHMARK_MAIN(); diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index be1d8768bd5..412eada7c0e 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2552,6 +2552,27 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "benchmark", + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_benchmark", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "bm_cq_multiple_threads", + "src": [ + "test/cpp/microbenchmarks/bm_cq_multiple_threads.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "benchmark", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 7d2c51a7d44..bda62794d23 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2737,6 +2737,28 @@ "posix" ] }, + { + "args": [ + "--benchmark_min_time=0" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_cq_multiple_threads", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, { "args": [ "--benchmark_min_time=0" From b7d833b865f43c808b3b68625ff7344b431b7da9 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 4 Apr 2017 14:16:09 -0700 Subject: [PATCH 64/90] Change filter_call_init_fails test to catch the bug. --- .../end2end/tests/filter_call_init_fails.c | 278 +++++++++++++++++- 1 file changed, 269 insertions(+), 9 deletions(-) diff --git a/test/core/end2end/tests/filter_call_init_fails.c b/test/core/end2end/tests/filter_call_init_fails.c index ebfe3b03dc3..2890d59a4c0 100644 --- a/test/core/end2end/tests/filter_call_init_fails.c +++ b/test/core/end2end/tests/filter_call_init_fails.c @@ -49,7 +49,9 @@ enum { TIMEOUT = 200000 }; -static bool g_enable_filter = false; +static bool g_enable_server_channel_filter = false; +static bool g_enable_client_channel_filter = false; +static bool g_enable_client_subchannel_filter = false; static void *tag(intptr_t t) { return (void *)t; } @@ -105,7 +107,7 @@ static void end_test(grpc_end2end_test_fixture *f) { // Simple request via a server filter that always fails to initialize // the call. -static void test_request(grpc_end2end_test_config config) { +static void test_server_channel_filter(grpc_end2end_test_config config) { grpc_call *c; grpc_call *s; grpc_slice request_payload_slice = @@ -199,6 +201,211 @@ static void test_request(grpc_end2end_test_config config) { config.tear_down_data(&f); } +// Simple request via a client channel filter that always fails to +// initialize the call. +static void test_client_channel_filter(grpc_end2end_test_config config) { + grpc_call *c; + grpc_slice request_payload_slice = + grpc_slice_from_copied_string("hello world"); + grpc_byte_buffer *request_payload = + grpc_raw_byte_buffer_create(&request_payload_slice, 1); + gpr_timespec deadline = five_seconds_time(); + grpc_end2end_test_fixture f = + begin_test(config, "filter_call_init_fails", NULL, NULL); + cq_verifier *cqv = cq_verifier_create(f.cq); + grpc_op ops[6]; + grpc_op *op; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_byte_buffer *request_payload_recv = NULL; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + grpc_slice details; + + c = grpc_channel_create_call( + f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, + grpc_slice_from_static_string("/foo"), + get_host_override_slice("foo.test.google.fr:1234", config), deadline, + NULL); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->data.send_initial_metadata.metadata = NULL; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message.send_message = request_payload; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->flags = 0; + op->reserved = NULL; + op++; + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(1), 1); + cq_verify(cqv); + + GPR_ASSERT(status == GRPC_STATUS_PERMISSION_DENIED); + GPR_ASSERT(0 == grpc_slice_str_cmp(details, "access denied")); + + grpc_slice_unref(details); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + + grpc_call_destroy(c); + + cq_verifier_destroy(cqv); + + grpc_byte_buffer_destroy(request_payload); + grpc_byte_buffer_destroy(request_payload_recv); + + end_test(&f); + config.tear_down_data(&f); +} + +// Simple request via a client subchannel filter that always fails to +// initialize the call. +static void test_client_subchannel_filter(grpc_end2end_test_config config) { + grpc_call *c; + grpc_slice request_payload_slice = + grpc_slice_from_copied_string("hello world"); + grpc_byte_buffer *request_payload = + grpc_raw_byte_buffer_create(&request_payload_slice, 1); + gpr_timespec deadline = five_seconds_time(); + grpc_end2end_test_fixture f = + begin_test(config, "filter_call_init_fails", NULL, NULL); + cq_verifier *cqv = cq_verifier_create(f.cq); + grpc_op ops[6]; + grpc_op *op; + grpc_metadata_array initial_metadata_recv; + grpc_metadata_array trailing_metadata_recv; + grpc_metadata_array request_metadata_recv; + grpc_byte_buffer *request_payload_recv = NULL; + grpc_call_details call_details; + grpc_status_code status; + grpc_call_error error; + grpc_slice details; + + c = grpc_channel_create_call( + f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, + grpc_slice_from_static_string("/foo"), + get_host_override_slice("foo.test.google.fr:1234", config), deadline, + NULL); + GPR_ASSERT(c); + + grpc_metadata_array_init(&initial_metadata_recv); + grpc_metadata_array_init(&trailing_metadata_recv); + grpc_metadata_array_init(&request_metadata_recv); + grpc_call_details_init(&call_details); + + memset(ops, 0, sizeof(ops)); + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->data.send_initial_metadata.metadata = NULL; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_MESSAGE; + op->data.send_message.send_message = request_payload; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv; + op->data.recv_status_on_client.status = &status; + op->data.recv_status_on_client.status_details = &details; + op->flags = 0; + op->reserved = NULL; + op++; + + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(1), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(1), 1); + cq_verify(cqv); + + GPR_ASSERT(status == GRPC_STATUS_PERMISSION_DENIED); + GPR_ASSERT(0 == grpc_slice_str_cmp(details, "access denied")); + + // Reset and create a new call. (The first call uses a different code + // path in client_channel.c than subsequent calls on the same channel, + // and we need to test both.) + grpc_call_destroy(c); + status = GRPC_STATUS_OK; + grpc_slice_unref(details); + details = grpc_empty_slice(); + + c = grpc_channel_create_call( + f.client, NULL, GRPC_PROPAGATE_DEFAULTS, f.cq, + grpc_slice_from_static_string("/foo"), + get_host_override_slice("foo.test.google.fr:1234", config), deadline, + NULL); + GPR_ASSERT(c); + + error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(2), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + + CQ_EXPECT_COMPLETION(cqv, tag(2), 1); + cq_verify(cqv); + + GPR_ASSERT(status == GRPC_STATUS_PERMISSION_DENIED); + GPR_ASSERT(0 == grpc_slice_str_cmp(details, "access denied")); + + grpc_slice_unref(details); + grpc_metadata_array_destroy(&initial_metadata_recv); + grpc_metadata_array_destroy(&trailing_metadata_recv); + grpc_metadata_array_destroy(&request_metadata_recv); + grpc_call_details_destroy(&call_details); + + grpc_call_destroy(c); + + cq_verifier_destroy(cqv); + + grpc_byte_buffer_destroy(request_payload); + grpc_byte_buffer_destroy(request_payload_recv); + + end_test(&f); + config.tear_down_data(&f); +} + /******************************************************************************* * Test filter - always fails to initialize a call */ @@ -242,9 +449,10 @@ static const grpc_channel_filter test_filter = { * Registration */ -static bool maybe_add_filter(grpc_exec_ctx *exec_ctx, - grpc_channel_stack_builder *builder, void *arg) { - if (g_enable_filter) { +static bool maybe_add_server_channel_filter(grpc_exec_ctx *exec_ctx, + grpc_channel_stack_builder *builder, + void *arg) { + if (g_enable_server_channel_filter) { // Want to add the filter as close to the end as possible, to make // sure that all of the filters work well together. However, we // can't add it at the very end, because the connected channel filter @@ -261,17 +469,69 @@ static bool maybe_add_filter(grpc_exec_ctx *exec_ctx, } } +static bool maybe_add_client_channel_filter(grpc_exec_ctx *exec_ctx, + grpc_channel_stack_builder *builder, + void *arg) { + if (g_enable_client_channel_filter) { + // Want to add the filter as close to the end as possible, to make + // sure that all of the filters work well together. However, we + // can't add it at the very end, because the connected channel filter + // must be the last one. So we add it right before the last one. + grpc_channel_stack_builder_iterator *it = + grpc_channel_stack_builder_create_iterator_at_last(builder); + GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); + const bool retval = grpc_channel_stack_builder_add_filter_before( + it, &test_filter, NULL, NULL); + grpc_channel_stack_builder_iterator_destroy(it); + return retval; + } else { + return true; + } +} + +static bool maybe_add_client_subchannel_filter( + grpc_exec_ctx *exec_ctx, grpc_channel_stack_builder *builder, void *arg) { + if (g_enable_client_subchannel_filter) { + // Want to add the filter as close to the end as possible, to make + // sure that all of the filters work well together. However, we + // can't add it at the very end, because the client channel filter + // must be the last one. So we add it right before the last one. + grpc_channel_stack_builder_iterator *it = + grpc_channel_stack_builder_create_iterator_at_last(builder); + GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); + const bool retval = grpc_channel_stack_builder_add_filter_before( + it, &test_filter, NULL, NULL); + grpc_channel_stack_builder_iterator_destroy(it); + return retval; + } else { + return true; + } +} + static void init_plugin(void) { grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, - maybe_add_filter, NULL); + maybe_add_server_channel_filter, NULL); + grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, + maybe_add_client_channel_filter, NULL); + grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX, + maybe_add_client_subchannel_filter, NULL); } static void destroy_plugin(void) {} void filter_call_init_fails(grpc_end2end_test_config config) { - g_enable_filter = true; - test_request(config); - g_enable_filter = false; + gpr_log(GPR_INFO, "Testing server channel filter."); + g_enable_server_channel_filter = true; + test_server_channel_filter(config); + g_enable_server_channel_filter = false; + gpr_log(GPR_INFO, "Testing client channel filter."); + g_enable_client_channel_filter = true; + test_client_channel_filter(config); + g_enable_client_channel_filter = false; + gpr_log(GPR_INFO, "Testing client subchannel filter."); + g_enable_client_subchannel_filter = true; + test_client_subchannel_filter(config); + g_enable_client_subchannel_filter = false; } void filter_call_init_fails_pre_init(void) { From 5bdffab4268977e035bb1ec8a75cd63b32f60183 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 4 Apr 2017 14:24:53 -0700 Subject: [PATCH 65/90] Add profiling annotations --- test/cpp/microbenchmarks/bm_call_create.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_call_create.cc b/test/cpp/microbenchmarks/bm_call_create.cc index 4af2263e823..2e597a5d83a 100644 --- a/test/cpp/microbenchmarks/bm_call_create.cc +++ b/test/cpp/microbenchmarks/bm_call_create.cc @@ -54,6 +54,7 @@ extern "C" { #include "src/core/lib/channel/http_client_filter.h" #include "src/core/lib/channel/http_server_filter.h" #include "src/core/lib/channel/message_size_filter.h" +#include "src/core/lib/profiling/timers.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/transport/transport_impl.h" } @@ -152,6 +153,7 @@ static void BM_LameChannelCallCreateCpp(benchmark::State &state) { grpc::testing::EchoResponse recv_response; grpc::Status recv_status; while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); grpc::ClientContext cli_ctx; auto reader = stub->AsyncEcho(&cli_ctx, send_request, &cq); reader->Finish(&recv_response, &recv_status, tag(0)); @@ -426,6 +428,7 @@ static void BM_IsolatedFilter(benchmark::State &state) { const int kArenaSize = 4096; call_args.arena = gpr_arena_create(kArenaSize); while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); GRPC_ERROR_UNREF(grpc_call_stack_init(&exec_ctx, channel_stack, 1, DoNothing, NULL, &call_args)); typename TestOp::Op op(&exec_ctx, &test_op_data, call_stack); @@ -590,6 +593,7 @@ static void BM_IsolatedCall_NoOp(benchmark::State &state) { void *method_hdl = grpc_channel_register_call(fixture.channel(), "/foo/bar", NULL, NULL); while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); grpc_call_destroy(grpc_channel_create_registered_call( fixture.channel(), nullptr, GRPC_PROPAGATE_DEFAULTS, fixture.cq(), method_hdl, deadline, NULL)); @@ -628,6 +632,7 @@ static void BM_IsolatedCall_Unary(benchmark::State &state) { ops[5].data.recv_status_on_client.status_details = &status_details; ops[5].data.recv_status_on_client.trailing_metadata = &recv_trailing_metadata; while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); grpc_call *call = grpc_channel_create_registered_call( fixture.channel(), nullptr, GRPC_PROPAGATE_DEFAULTS, fixture.cq(), method_hdl, deadline, NULL); @@ -670,6 +675,7 @@ static void BM_IsolatedCall_StreamingSend(benchmark::State &state) { ops[0].op = GRPC_OP_SEND_MESSAGE; ops[0].data.send_message.send_message = send_message; while (state.KeepRunning()) { + GPR_TIMER_SCOPE("BenchmarkCycle", 0); grpc_call_start_batch(call, ops, 1, tag(2), NULL); grpc_completion_queue_next(fixture.cq(), gpr_inf_future(GPR_CLOCK_MONOTONIC), NULL); From e800e1c5d0263fb80896cd70b7af9bf84068561e Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 4 Apr 2017 14:51:37 -0700 Subject: [PATCH 66/90] Fix the hard conversion problem in cronet_transport.c --- src/core/ext/transport/cronet/transport/cronet_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 9bd8914b988..0b9189558f8 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -1178,7 +1178,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, if (stream_state->rs.compressed) { stream_state->rs.sbs.base.flags = GRPC_WRITE_INTERNAL_COMPRESS; } - *((grpc_byte_buffer **)stream_op->recv_message) = + *((grpc_byte_buffer **)stream_op->payload->recv_message.recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; grpc_closure_sched(exec_ctx, stream_op->payload->recv_message.recv_message_ready, From d0ee7a92baa0791083e03dd55745b7a7c5c51e88 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Tue, 4 Apr 2017 14:51:37 -0700 Subject: [PATCH 67/90] Fix the hard conversion problem in cronet_transport.c --- src/core/ext/transport/cronet/transport/cronet_transport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 9bd8914b988..0b9189558f8 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -1178,7 +1178,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, if (stream_state->rs.compressed) { stream_state->rs.sbs.base.flags = GRPC_WRITE_INTERNAL_COMPRESS; } - *((grpc_byte_buffer **)stream_op->recv_message) = + *((grpc_byte_buffer **)stream_op->payload->recv_message.recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; grpc_closure_sched(exec_ctx, stream_op->payload->recv_message.recv_message_ready, From 2c3e7e80bd861d28e6395f503650828c0eb05580 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 4 Apr 2017 16:21:52 -0700 Subject: [PATCH 68/90] use wallclock --- test/cpp/microbenchmarks/bm_cq_multiple_threads.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index f68baad2b01..82f402d3ddd 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -127,18 +127,16 @@ static void BM_Cq_Throughput(benchmark::State& state) { if (state.thread_index == 0) { grpc_completion_queue_shutdown(g_cq); grpc_completion_queue_destroy(g_cq); - - state.SetItemsProcessed(state.iterations()); } track_counters.Finish(state); } -BENCHMARK(BM_Cq_Throughput)->Threads(1); -BENCHMARK(BM_Cq_Throughput)->Threads(2); -BENCHMARK(BM_Cq_Throughput)->Threads(4); -BENCHMARK(BM_Cq_Throughput)->Threads(8); -BENCHMARK(BM_Cq_Throughput)->Threads(16); +BENCHMARK(BM_Cq_Throughput)->Threads(1)->UseRealTime(); +BENCHMARK(BM_Cq_Throughput)->Threads(2)->UseRealTime(); +BENCHMARK(BM_Cq_Throughput)->Threads(4)->UseRealTime(); +BENCHMARK(BM_Cq_Throughput)->Threads(8)->UseRealTime(); +BENCHMARK(BM_Cq_Throughput)->Threads(16)->UseRealTime(); } // namespace testing } // namespace grpc From e4e763265bdf9e5a4d6098b3cb5380b00f3bd29f Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 4 Apr 2017 16:28:59 -0700 Subject: [PATCH 69/90] Add UpdateActions to the interop client helper --- test/cpp/interop/client.cc | 3 +++ test/cpp/interop/client_helper.cc | 3 +++ test/cpp/interop/client_helper.h | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/test/cpp/interop/client.cc b/test/cpp/interop/client.cc index 5688ab79716..369413e6a1d 100644 --- a/test/cpp/interop/client.cc +++ b/test/cpp/interop/client.cc @@ -99,6 +99,7 @@ DEFINE_bool(do_not_abort_on_transient_failures, false, using grpc::testing::CreateChannelForTestCase; using grpc::testing::GetServiceAccountJsonKey; +using grpc::testing::UpdateActions; int main(int argc, char** argv) { grpc::testing::InitTest(&argc, &argv, true); @@ -165,6 +166,8 @@ int main(int argc, char** argv) { // actions["cacheable_unary"] = // std::bind(&grpc::testing::InteropClient::DoCacheableUnary, &client); + UpdateActions(&actions); + if (FLAGS_test_case == "all") { for (const auto& action : actions) { action.second(); diff --git a/test/cpp/interop/client_helper.cc b/test/cpp/interop/client_helper.cc index d3192ad0c93..784cd2826d6 100644 --- a/test/cpp/interop/client_helper.cc +++ b/test/cpp/interop/client_helper.cc @@ -89,6 +89,9 @@ grpc::string GetOauth2AccessToken() { return access_token; } +void UpdateActions( + std::unordered_map>* actions) {} + std::shared_ptr CreateChannelForTestCase( const grpc::string& test_case) { GPR_ASSERT(FLAGS_server_port); diff --git a/test/cpp/interop/client_helper.h b/test/cpp/interop/client_helper.h index 622b96e4fbf..387530a21c7 100644 --- a/test/cpp/interop/client_helper.h +++ b/test/cpp/interop/client_helper.h @@ -35,6 +35,7 @@ #define GRPC_TEST_CPP_INTEROP_CLIENT_HELPER_H #include +#include #include @@ -47,6 +48,9 @@ grpc::string GetServiceAccountJsonKey(); grpc::string GetOauth2AccessToken(); +void UpdateActions( + std::unordered_map>* actions); + std::shared_ptr CreateChannelForTestCase( const grpc::string& test_case); From 0cbfad62cf087b9170f6b3aee9925161de94e865 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Tue, 4 Apr 2017 12:41:06 -0700 Subject: [PATCH 70/90] Support custom pool in enable_server_reflection --- .../grpcio_reflection/grpc_reflection/v1alpha/reflection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py index c8ad9668acc..c58cb3ecf17 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py @@ -143,12 +143,13 @@ class ReflectionServicer(reflection_pb2.ServerReflectionServicer): .encode(),)) -def enable_server_reflection(service_names, server): +def enable_server_reflection(service_names, server, pool=None): """Enables server reflection on a server. Args: service_names: Iterable of fully-qualified service names available. server: grpc.Server to which reflection service will be added. + pool: DescriptorPool object to use (descriptor_pool.Default() if None). """ reflection_pb2_grpc.add_ServerReflectionServicer_to_server( - ReflectionServicer(service_names), server) + ReflectionServicer(service_names), server, pool) From f712ae76f4ecf032c7be2f880d5193d5d4c35332 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 4 Apr 2017 16:58:05 -0700 Subject: [PATCH 71/90] Fix a couple of issues with the use of the Protobuf.js API --- src/node/src/protobuf_js_6_common.js | 4 ++-- src/node/test/common_test.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/node/src/protobuf_js_6_common.js b/src/node/src/protobuf_js_6_common.js index cac0f711451..21eddabaa8a 100644 --- a/src/node/src/protobuf_js_6_common.js +++ b/src/node/src/protobuf_js_6_common.js @@ -77,11 +77,11 @@ exports.serializeCls = function serializeCls(cls) { * @return {Buffer} The serialized object */ return function serialize(arg) { - var message = cls.fromObject(arg); - var errMsg = cls.verify(message); + var errMsg = cls.verify(arg); if (errMsg) { throw Error(errMsg); } + var message = cls.create(arg); return cls.encode(message).finish(); }; }; diff --git a/src/node/test/common_test.js b/src/node/test/common_test.js index 39ff6a5f1fc..e1ce864f975 100644 --- a/src/node/test/common_test.js +++ b/src/node/test/common_test.js @@ -176,7 +176,7 @@ describe('Proto message oneof serialize and deserialize', function() { var test_message2 = {oneof_choice: 'string_choice', string_choice: 'abc'}; var serialized2 = oneofSerialize(test_message2); var deserialized2 = oneofDeserialize(serialized2); - assert.equal(deserialized2.oneof_choice, 'int_choice'); + assert.equal(deserialized2.oneof_choice, 'string_choice'); }); }); describe('Proto message enum serialize and deserialize', function() { From f7c673a8c789e6b521b2b83c82eb433900fd4b78 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 4 Apr 2017 22:52:44 -0700 Subject: [PATCH 72/90] minor change --- test/cpp/microbenchmarks/bm_cq_multiple_threads.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index 82f402d3ddd..80f21f1f697 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -132,11 +132,7 @@ static void BM_Cq_Throughput(benchmark::State& state) { track_counters.Finish(state); } -BENCHMARK(BM_Cq_Throughput)->Threads(1)->UseRealTime(); -BENCHMARK(BM_Cq_Throughput)->Threads(2)->UseRealTime(); -BENCHMARK(BM_Cq_Throughput)->Threads(4)->UseRealTime(); -BENCHMARK(BM_Cq_Throughput)->Threads(8)->UseRealTime(); -BENCHMARK(BM_Cq_Throughput)->Threads(16)->UseRealTime(); +BENCHMARK(BM_Cq_Throughput)->ThreadRange(1, 16)->UseRealTime(); } // namespace testing } // namespace grpc From 9e0b79e047988e9b3bff6a995a5af8e3553d03e2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 06:43:37 -0700 Subject: [PATCH 73/90] Update to new API --- src/core/lib/iomgr/endpoint_pair_uv.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/lib/iomgr/endpoint_pair_uv.c b/src/core/lib/iomgr/endpoint_pair_uv.c index ff24894c6db..9718eb05237 100644 --- a/src/core/lib/iomgr/endpoint_pair_uv.c +++ b/src/core/lib/iomgr/endpoint_pair_uv.c @@ -41,9 +41,8 @@ #include "src/core/lib/iomgr/endpoint_pair.h" -grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( - const char *name, grpc_resource_quota *resource_quota, - size_t read_slice_size) { +grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name, + grpc_channel_args *args) { grpc_endpoint_pair endpoint_pair; // TODO(mlumish): implement this properly under libuv GPR_ASSERT(false && From e645d13bda9cca7e4b7a89e78a9d8f2450377d3a Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 5 Apr 2017 06:47:35 -0700 Subject: [PATCH 74/90] Fix build. --- test/core/end2end/tests/filter_call_init_fails.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/core/end2end/tests/filter_call_init_fails.c b/test/core/end2end/tests/filter_call_init_fails.c index 3eba4920c29..bcb5e2d3ff9 100644 --- a/test/core/end2end/tests/filter_call_init_fails.c +++ b/test/core/end2end/tests/filter_call_init_fails.c @@ -107,8 +107,8 @@ static void end_test(grpc_end2end_test_fixture *f) { grpc_completion_queue_destroy(f->cq); } -// Simple request via a server filter that always fails to initialize -// the call. +// Simple request via a server channel filter that always fails to +// initialize the call. static void test_server_channel_filter(grpc_end2end_test_config config) { grpc_call *c; grpc_call *s; @@ -211,7 +211,7 @@ static void test_client_channel_filter(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); + gpr_timespec deadline = five_seconds_from_now(); grpc_end2end_test_fixture f = begin_test(config, "filter_call_init_fails", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); @@ -301,7 +301,7 @@ static void test_client_subchannel_filter(grpc_end2end_test_config config) { grpc_slice_from_copied_string("hello world"); grpc_byte_buffer *request_payload = grpc_raw_byte_buffer_create(&request_payload_slice, 1); - gpr_timespec deadline = five_seconds_time(); + gpr_timespec deadline = five_seconds_from_now(); grpc_end2end_test_fixture f = begin_test(config, "filter_call_init_fails", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); From 358f1024ea1afaa12e923296cab7e74082f52b7a Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Wed, 5 Apr 2017 08:42:21 -0700 Subject: [PATCH 75/90] backport #10100 and #10377 --- .../grpc_interop_http2/Dockerfile.template | 2 +- test/http2_test/http2_server_health_check.py | 49 ++++++++++++++ test/http2_test/http2_test_server.py | 21 +++++- .../interoptest/grpc_interop_http2/Dockerfile | 2 +- tools/run_tests/python_utils/dockerjob.py | 20 ++++++ tools/run_tests/run_interop_tests.py | 66 +++++++++++-------- 6 files changed, 129 insertions(+), 31 deletions(-) create mode 100644 test/http2_test/http2_server_health_check.py diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template index 0899e007d9e..b517921f087 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template @@ -33,7 +33,7 @@ <%include file="../../go_path.include"/> <%include file="../../python_deps.include"/> - RUN pip install twisted h2 + RUN pip install twisted h2==2.6.1 hyper # Define the default command. CMD ["bash"] diff --git a/test/http2_test/http2_server_health_check.py b/test/http2_test/http2_server_health_check.py new file mode 100644 index 00000000000..dd9402b8557 --- /dev/null +++ b/test/http2_test/http2_server_health_check.py @@ -0,0 +1,49 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import hyper +import sys + +# Utility to healthcheck the http2 server. Used when starting the server to +# verify that the server is live before tests begin. +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--server_host', type=str, default='localhost') + parser.add_argument('--server_port', type=int, default=8080) + args = parser.parse_args() + server_host = args.server_host + server_port = args.server_port + conn = hyper.HTTP20Connection('%s:%d' % (server_host, server_port)) + conn.request('POST', '/grpc.testing.TestService/UnaryCall') + resp = conn.get_response() + if resp.headers.get('grpc-encoding') is None: + sys.exit(1) + else: + sys.exit(0) diff --git a/test/http2_test/http2_test_server.py b/test/http2_test/http2_test_server.py index abde3433ad2..46c3e00d18f 100644 --- a/test/http2_test/http2_test_server.py +++ b/test/http2_test/http2_test_server.py @@ -31,6 +31,7 @@ import argparse import logging +import sys import twisted import twisted.internet import twisted.internet.endpoints @@ -53,9 +54,11 @@ _TEST_CASE_MAPPING = { 'max_streams': test_max_streams.TestcaseSettingsMaxStreams, } +_exit_code = 0 + class H2Factory(twisted.internet.protocol.Factory): def __init__(self, testcase): - logging.info('Creating H2Factory for new connection.') + logging.info('Creating H2Factory for new connection (%s)', testcase) self._num_streams = 0 self._testcase = testcase @@ -83,6 +86,17 @@ def parse_arguments(): ) return parser.parse_args() +def listen(endpoint, test_case): + deferred = endpoint.listen(H2Factory(test_case)) + def listen_error(reason): + # If listening fails, we stop the reactor and exit the program + # with exit code 1. + global _exit_code + _exit_code = 1 + logging.error('Listening failed: %s' % reason.value) + twisted.internet.reactor.stop() + deferred.addErrback(listen_error) + def start_test_servers(base_port): """ Start one server per test case on incrementing port numbers beginning with base_port """ @@ -92,7 +106,9 @@ def start_test_servers(base_port): logging.warning('serving on port %d : %s'%(portnum, test_case)) endpoint = twisted.internet.endpoints.TCP4ServerEndpoint( twisted.internet.reactor, portnum, backlog=128) - endpoint.listen(H2Factory(test_case)) + # Wait until the reactor is running before calling endpoint.listen(). + twisted.internet.reactor.callWhenRunning(listen, endpoint, test_case) + index += 1 if __name__ == '__main__': @@ -102,3 +118,4 @@ if __name__ == '__main__': args = parse_arguments() start_test_servers(args.base_port) twisted.internet.reactor.run() + sys.exit(_exit_code) diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 42e9edf98ad..094a4e096df 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -47,7 +47,7 @@ RUN pip install pip --upgrade RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 -RUN pip install twisted h2 +RUN pip install twisted h2==2.6.1 hyper # Define the default command. CMD ["bash"] diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index 0869c5cee9b..709fc121a9b 100755 --- a/tools/run_tests/python_utils/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -70,6 +70,23 @@ def docker_mapped_port(cid, port, timeout_seconds=15): (port, cid)) +def wait_for_healthy(cid, shortname, timeout_seconds): + """Wait timeout_seconds for the container to become healthy""" + started = time.time() + while time.time() - started < timeout_seconds: + try: + output = subprocess.check_output( + ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid], + stderr=_DEVNULL) + if output.strip('\n') == 'healthy': + return + except subprocess.CalledProcessError as e: + pass + time.sleep(1) + raise Exception('Timed out waiting for %s (%s) to pass health check' % + (shortname, cid)) + + def finish_jobs(jobs): """Kills given docker containers and waits for corresponding jobs to finish""" for job in jobs: @@ -113,6 +130,9 @@ class DockerJob: def mapped_port(self, port): return docker_mapped_port(self._container_name, port) + def wait_for_healthy(self, timeout_seconds): + wait_for_healthy(self._container_name, self._spec.shortname, timeout_seconds) + def kill(self, suppress_failure=False): """Sends kill signal to the container.""" if suppress_failure: diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 0d5bec1d67f..007e341ca96 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -607,17 +607,13 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, common_options = [ '--test_case=%s' % test_case, '--server_host=%s' % server_host, + '--server_port=%s' % server_port, ] if test_case in _HTTP2_BADSERVER_TEST_CASES: - # We are running the http2_badserver_interop test. Adjust command line accordingly. - offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) - client_options = common_options + ['--server_port=%s' % - (int(server_port)+offset)] - cmdline = bash_cmdline(language.client_cmd_http2interop(client_options)) + cmdline = bash_cmdline(language.client_cmd_http2interop(common_options)) cwd = language.http2_cwd else: - client_options = interop_only_options + common_options + ['--server_port=%s' % server_port] - cmdline = bash_cmdline(language.client_cmd(client_options)) + cmdline = bash_cmdline(language.client_cmd(common_options+interop_only_options)) cwd = language.client_cwd environ = language.global_env() @@ -653,30 +649,40 @@ def server_jobspec(language, docker_image, insecure=False): language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT, '--use_tls=%s' % ('false' if insecure else 'true')])) environ = language.global_env() + docker_args = ['--name=%s' % container_name] if language.safename == 'http2': # we are running the http2 interop server. Open next N ports beginning # with the server port. These ports are used for http2 interop test - # (one test case per port). We also attach the docker container running - # the server to local network, so we don't have to mess with port mapping - port_args = [ - '-p', str(_DEFAULT_SERVER_PORT+0), - '-p', str(_DEFAULT_SERVER_PORT+1), - '-p', str(_DEFAULT_SERVER_PORT+2), - '-p', str(_DEFAULT_SERVER_PORT+3), - '-p', str(_DEFAULT_SERVER_PORT+4), - '-p', str(_DEFAULT_SERVER_PORT+5), - '-p', str(_DEFAULT_SERVER_PORT+6), - '--net=host', + # (one test case per port). + docker_args += list( + itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i)) + for i in range( + len(_HTTP2_BADSERVER_TEST_CASES)))) + # Enable docker's healthcheck mechanism. + # This runs a Python script inside the container every second. The script + # pings the http2 server to verify it is ready. The 'health-retries' flag + # specifies the number of consecutive failures before docker will report + # the container's status as 'unhealthy'. Prior to the first 'health_retries' + # failures or the first success, the status will be 'starting'. 'docker ps' + # or 'docker inspect' can be used to see the health of the container on the + # command line. + docker_args += [ + '--health-cmd=python test/http2_test/http2_server_health_check.py ' + '--server_host=%s --server_port=%d' + % ('localhost', _DEFAULT_SERVER_PORT), + '--health-interval=1s', + '--health-retries=5', + '--health-timeout=10s', ] + else: - port_args = ['-p', str(_DEFAULT_SERVER_PORT)] + docker_args += ['-p', str(_DEFAULT_SERVER_PORT)] docker_cmdline = docker_run_cmdline(cmdline, image=docker_image, cwd=language.server_cwd, environ=environ, - docker_args=port_args + - ['--name', container_name]) + docker_args=docker_args) server_job = jobset.JobSpec( cmdline=docker_cmdline, environ=environ, @@ -849,7 +855,8 @@ languages = set(_LANGUAGES[l] languages_http2_badserver_interop = set() if args.http2_badserver_interop: languages_http2_badserver_interop = set( - _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS) + _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS + if 'all' in args.language or l in args.language) http2Interop = Http2Client() if args.http2_interop else None http2InteropServer = Http2Server() if args.http2_badserver_interop else None @@ -888,8 +895,9 @@ if args.use_docker: sys.exit(1) # Start interop servers. -server_jobs={} -server_addresses={} +server_jobs = {} +server_addresses = {} +http2_badserver_ports = () try: for s in servers: lang = str(s) @@ -899,12 +907,13 @@ try: server_jobs[lang] = job server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT)) + http2_badserver_job = None if args.http2_badserver_interop: # launch a HTTP2 server emulator that creates edge cases lang = str(http2InteropServer) spec = server_jobspec(http2InteropServer, docker_images.get(lang)) - job = dockerjob.DockerJob(spec) - server_jobs[lang] = job + http2_badserver_job = dockerjob.DockerJob(spec) + server_jobs[lang] = http2_badserver_job jobs = [] if args.cloud_to_prod: @@ -981,13 +990,16 @@ try: jobs.append(test_job) if args.http2_badserver_interop: + http2_badserver_job.wait_for_healthy(timeout_seconds=600) for language in languages_http2_badserver_interop: for test_case in _HTTP2_BADSERVER_TEST_CASES: + offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case) + server_port = http2_badserver_job.mapped_port(_DEFAULT_SERVER_PORT+offset) test_job = cloud_to_cloud_jobspec(language, test_case, str(http2InteropServer), 'localhost', - _DEFAULT_SERVER_PORT, + server_port, docker_image=docker_images.get(str(language))) jobs.append(test_job) From b39bf85dfce99804a4e07991a36431696a43d10c Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 5 Apr 2017 08:55:37 -0700 Subject: [PATCH 76/90] Fix test for CLIENT_DIRECT_CHANNEL fixtures. --- .../end2end/tests/filter_call_init_fails.c | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/test/core/end2end/tests/filter_call_init_fails.c b/test/core/end2end/tests/filter_call_init_fails.c index bcb5e2d3ff9..0c4f0dd42fe 100644 --- a/test/core/end2end/tests/filter_call_init_fails.c +++ b/test/core/end2end/tests/filter_call_init_fails.c @@ -107,7 +107,7 @@ static void end_test(grpc_end2end_test_fixture *f) { grpc_completion_queue_destroy(f->cq); } -// Simple request via a server channel filter that always fails to +// Simple request via a SERVER_CHANNEL filter that always fails to // initialize the call. static void test_server_channel_filter(grpc_end2end_test_config config) { grpc_call *c; @@ -203,8 +203,8 @@ static void test_server_channel_filter(grpc_end2end_test_config config) { config.tear_down_data(&f); } -// Simple request via a client channel filter that always fails to -// initialize the call. +// Simple request via a CLIENT_CHANNEL or CLIENT_DIRECT_CHANNEL filter +// that always fails to initialize the call. static void test_client_channel_filter(grpc_end2end_test_config config) { grpc_call *c; grpc_slice request_payload_slice = @@ -293,7 +293,7 @@ static void test_client_channel_filter(grpc_end2end_test_config config) { config.tear_down_data(&f); } -// Simple request via a client subchannel filter that always fails to +// Simple request via a CLIENT_SUBCHANNEL filter that always fails to // initialize the call. static void test_client_subchannel_filter(grpc_end2end_test_config config) { grpc_call *c; @@ -517,23 +517,27 @@ static void init_plugin(void) { maybe_add_client_channel_filter, NULL); grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX, maybe_add_client_subchannel_filter, NULL); + grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, + maybe_add_client_channel_filter, NULL); } static void destroy_plugin(void) {} void filter_call_init_fails(grpc_end2end_test_config config) { - gpr_log(GPR_INFO, "Testing server channel filter."); + gpr_log(GPR_INFO, "Testing SERVER_CHANNEL filter."); g_enable_server_channel_filter = true; test_server_channel_filter(config); g_enable_server_channel_filter = false; - gpr_log(GPR_INFO, "Testing client channel filter."); + gpr_log(GPR_INFO, "Testing CLIENT_CHANNEL / CLIENT_DIRECT_CHANNEL filter."); g_enable_client_channel_filter = true; test_client_channel_filter(config); g_enable_client_channel_filter = false; - gpr_log(GPR_INFO, "Testing client subchannel filter."); - g_enable_client_subchannel_filter = true; - test_client_subchannel_filter(config); - g_enable_client_subchannel_filter = false; + if (config.feature_mask & FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL) { + gpr_log(GPR_INFO, "Testing CLIENT_SUBCHANNEL filter."); + g_enable_client_subchannel_filter = true; + test_client_subchannel_filter(config); + g_enable_client_subchannel_filter = false; + } } void filter_call_init_fails_pre_init(void) { From 61a6398675ae3afd3ee305a4bba3c6c148939a05 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 5 Apr 2017 09:56:12 -0700 Subject: [PATCH 77/90] Fix refcounting for subchannel call stack init failures. --- .../ext/filters/client_channel/client_channel.c | 15 ++++++--------- src/core/ext/filters/client_channel/subchannel.c | 3 +-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index 04b0ffdaf3d..83e3b8f118d 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -914,14 +914,14 @@ static void subchannel_ready_locked(grpc_exec_ctx *exec_ctx, void *arg, .arena = calld->arena}; grpc_error *new_error = grpc_connected_subchannel_create_call( exec_ctx, calld->connected_subchannel, &call_args, &subchannel_call); + gpr_atm_rel_store(&calld->subchannel_call, + (gpr_atm)(uintptr_t)subchannel_call); if (new_error != GRPC_ERROR_NONE) { new_error = grpc_error_add_child(new_error, error); - subchannel_call = CANCELLED_CALL; fail_locked(exec_ctx, calld, new_error); + } else { + retry_waiting_locked(exec_ctx, calld); } - gpr_atm_rel_store(&calld->subchannel_call, - (gpr_atm)(uintptr_t)subchannel_call); - retry_waiting_locked(exec_ctx, calld); } GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_subchannel"); } @@ -1152,15 +1152,12 @@ static void start_transport_stream_op_batch_locked_inner( .arena = calld->arena}; grpc_error *error = grpc_connected_subchannel_create_call( exec_ctx, calld->connected_subchannel, &call_args, &subchannel_call); + gpr_atm_rel_store(&calld->subchannel_call, + (gpr_atm)(uintptr_t)subchannel_call); if (error != GRPC_ERROR_NONE) { - subchannel_call = CANCELLED_CALL; - gpr_atm_rel_store(&calld->subchannel_call, - (gpr_atm)(uintptr_t)subchannel_call); fail_locked(exec_ctx, calld, GRPC_ERROR_REF(error)); grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error); } else { - gpr_atm_rel_store(&calld->subchannel_call, - (gpr_atm)(uintptr_t)subchannel_call); retry_waiting_locked(exec_ctx, calld); /* recurse to retry */ start_transport_stream_op_batch_locked_inner(exec_ctx, op, elem); diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c index 29a1a095553..9a7a7a0ee5f 100644 --- a/src/core/ext/filters/client_channel/subchannel.c +++ b/src/core/ext/filters/client_channel/subchannel.c @@ -769,7 +769,7 @@ grpc_error *grpc_connected_subchannel_create_call( *call = gpr_arena_alloc( args->arena, sizeof(grpc_subchannel_call) + chanstk->call_stack_size); grpc_call_stack *callstk = SUBCHANNEL_CALL_TO_CALL_STACK(*call); - (*call)->connection = con; // Ref is added below. + (*call)->connection = GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call"); const grpc_call_element_args call_args = {.call_stack = callstk, .server_transport_data = NULL, .context = NULL, @@ -784,7 +784,6 @@ grpc_error *grpc_connected_subchannel_create_call( gpr_log(GPR_ERROR, "error: %s", error_string); return error; } - GRPC_CONNECTED_SUBCHANNEL_REF(con, "subchannel_call"); grpc_call_stack_set_pollset_or_pollset_set(exec_ctx, callstk, args->pollent); return GRPC_ERROR_NONE; } From de0906553097e24c91def857c0227f281d347b7c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 14:00:24 -0700 Subject: [PATCH 78/90] Update windows --- src/core/lib/iomgr/endpoint_pair_windows.c | 13 +++++------ src/core/lib/iomgr/tcp_client_windows.c | 20 ++++------------- src/core/lib/iomgr/tcp_server_windows.c | 25 ++++++---------------- src/core/lib/iomgr/tcp_windows.c | 17 ++++++++++++--- src/core/lib/iomgr/tcp_windows.h | 4 ++-- 5 files changed, 33 insertions(+), 46 deletions(-) diff --git a/src/core/lib/iomgr/endpoint_pair_windows.c b/src/core/lib/iomgr/endpoint_pair_windows.c index 93f71b745c6..dcf8166104e 100644 --- a/src/core/lib/iomgr/endpoint_pair_windows.c +++ b/src/core/lib/iomgr/endpoint_pair_windows.c @@ -83,15 +83,16 @@ static void create_sockets(SOCKET sv[2]) { } grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( - const char *name, grpc_resource_quota *resource_quota, - size_t read_slice_size) { + const char *name, grpc_channel_args *channel_args) { SOCKET sv[2]; grpc_endpoint_pair p; create_sockets(sv); - p.client = grpc_tcp_create(grpc_winsocket_create(sv[1], "endpoint:client"), - resource_quota, "endpoint:server"); - p.server = grpc_tcp_create(grpc_winsocket_create(sv[0], "endpoint:server"), - resource_quota, "endpoint:client"); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + p.client = grpc_tcp_create(&exec_ctx, grpc_winsocket_create(sv[1], "endpoint:client"), + channel_args, "endpoint:server"); + p.server = grpc_tcp_create(&exec_ctx,grpc_winsocket_create(sv[0], "endpoint:server"), + channel_args, "endpoint:client"); + grpc_exec_ctx_finish(&exec_ctx); return p; } diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c index a356564766a..935f96b810e 100644 --- a/src/core/lib/iomgr/tcp_client_windows.c +++ b/src/core/lib/iomgr/tcp_client_windows.c @@ -63,7 +63,7 @@ typedef struct { int refs; grpc_closure on_connect; grpc_endpoint **endpoint; - grpc_resource_quota *resource_quota; + grpc_channel_args *channel_args; } async_connect; static void async_connect_unlock_and_cleanup(grpc_exec_ctx *exec_ctx, @@ -72,7 +72,7 @@ static void async_connect_unlock_and_cleanup(grpc_exec_ctx *exec_ctx, int done = (--ac->refs == 0); gpr_mu_unlock(&ac->mu); if (done) { - grpc_resource_quota_unref_internal(exec_ctx, ac->resource_quota); + grpc_channel_args_destroy(exec_ctx, ac->channel_args); gpr_mu_destroy(&ac->mu); gpr_free(ac->addr_name); gpr_free(ac); @@ -119,7 +119,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { if (!wsa_success) { error = GRPC_WSA_ERROR(WSAGetLastError(), "ConnectEx"); } else { - *ep = grpc_tcp_create(socket, ac->resource_quota, ac->addr_name); + *ep = grpc_tcp_create(exec_ctx, socket, ac->channel_args, ac->addr_name); socket = NULL; } } else { @@ -152,17 +152,6 @@ static void tcp_client_connect_impl( grpc_winsocket_callback_info *info; grpc_error *error = GRPC_ERROR_NONE; - grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); - if (channel_args != NULL) { - for (size_t i = 0; i < channel_args->num_args; i++) { - if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { - grpc_resource_quota_unref_internal(exec_ctx, resource_quota); - resource_quota = grpc_resource_quota_ref_internal( - channel_args->args[i].value.pointer.p); - } - } - } - *endpoint = NULL; /* Use dualstack sockets where available. */ @@ -225,7 +214,7 @@ static void tcp_client_connect_impl( ac->refs = 2; ac->addr_name = grpc_sockaddr_to_uri(addr); ac->endpoint = endpoint; - ac->resource_quota = resource_quota; + ac->channel_args = grpc_channel_args_copy(channel_args); grpc_closure_init(&ac->on_connect, on_connect, ac, grpc_schedule_on_exec_ctx); grpc_closure_init(&ac->on_alarm, on_alarm, ac, grpc_schedule_on_exec_ctx); @@ -247,7 +236,6 @@ failure: } else if (sock != INVALID_SOCKET) { closesocket(sock); } - grpc_resource_quota_unref_internal(exec_ctx, resource_quota); grpc_closure_sched(exec_ctx, on_done, final_error); } diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c index 12ce7d3fdd5..3b077e6d1ff 100644 --- a/src/core/lib/iomgr/tcp_server_windows.c +++ b/src/core/lib/iomgr/tcp_server_windows.c @@ -53,6 +53,7 @@ #include "src/core/lib/iomgr/socket_windows.h" #include "src/core/lib/iomgr/tcp_server.h" #include "src/core/lib/iomgr/tcp_windows.h" +#include "src/core/lib/channel/channel_args.h" #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100 @@ -102,7 +103,7 @@ struct grpc_tcp_server { /* shutdown callback */ grpc_closure *shutdown_complete; - grpc_resource_quota *resource_quota; + grpc_channel_args *channel_args; }; /* Public function. Allocates the proper data structures to hold a @@ -112,21 +113,7 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx, const grpc_channel_args *args, grpc_tcp_server **server) { grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server)); - s->resource_quota = grpc_resource_quota_create(NULL); - for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) { - if (0 == strcmp(GRPC_ARG_RESOURCE_QUOTA, args->args[i].key)) { - if (args->args[i].type == GRPC_ARG_POINTER) { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); - s->resource_quota = - grpc_resource_quota_ref_internal(args->args[i].value.pointer.p); - } else { - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); - gpr_free(s); - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - GRPC_ARG_RESOURCE_QUOTA " must be a pointer to a buffer pool"); - } - } - } + s->channel_args = grpc_channel_args_copy(args); gpr_ref_init(&s->refs, 1); gpr_mu_init(&s->mu); s->active_ports = 0; @@ -155,7 +142,7 @@ static void destroy_server(grpc_exec_ctx *exec_ctx, void *arg, grpc_winsocket_destroy(sp->socket); gpr_free(sp); } - grpc_resource_quota_unref_internal(exec_ctx, s->resource_quota); + grpc_channel_args_destroy(exec_ctx, s->channel_args); gpr_free(s); } @@ -383,8 +370,8 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { gpr_free(utf8_message); } gpr_asprintf(&fd_name, "tcp_server:%s", peer_name_string); - ep = grpc_tcp_create(grpc_winsocket_create(sock, fd_name), - sp->server->resource_quota, peer_name_string); + ep = grpc_tcp_create(exec_ctx, grpc_winsocket_create(sock, fd_name), + sp->server->channel_args, peer_name_string); gpr_free(fd_name); gpr_free(peer_name_string); } else { diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c index 91348832268..35b991cb127 100644 --- a/src/core/lib/iomgr/tcp_windows.c +++ b/src/core/lib/iomgr/tcp_windows.c @@ -430,10 +430,21 @@ static grpc_endpoint_vtable vtable = {win_read, win_get_peer, win_get_fd}; -grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, - grpc_resource_quota *resource_quota, +grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket, + grpc_channel_args *channel_args, char *peer_string) { - grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); + grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); + if (channel_args != NULL) { + for (size_t i = 0; i < channel_args->num_args; i++) { +if (0 == + strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { + grpc_resource_quota_unref_internal(exec_ctx, resource_quota); + resource_quota = grpc_resource_quota_ref_internal( + channel_args->args[i].value.pointer.p); + } + } + } + grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); memset(tcp, 0, sizeof(grpc_tcp)); tcp->base.vtable = &vtable; tcp->socket = socket; diff --git a/src/core/lib/iomgr/tcp_windows.h b/src/core/lib/iomgr/tcp_windows.h index 4402de1c385..abafdb22d20 100644 --- a/src/core/lib/iomgr/tcp_windows.h +++ b/src/core/lib/iomgr/tcp_windows.h @@ -50,8 +50,8 @@ /* Create a tcp endpoint given a winsock handle. * Takes ownership of the handle. */ -grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, - grpc_resource_quota *resource_quota, +grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket, + grpc_channel_args *channel_args, char *peer_string); grpc_error *grpc_tcp_prepare_socket(SOCKET sock); From 4254d3bc31eb12487012fe3571ea22851f238524 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 14:03:49 -0700 Subject: [PATCH 79/90] clang-format --- src/core/lib/iomgr/endpoint_pair_windows.c | 8 +++++--- src/core/lib/iomgr/tcp_client_windows.c | 5 +++-- src/core/lib/iomgr/tcp_server_windows.c | 2 +- src/core/lib/iomgr/tcp_windows.c | 23 +++++++++++----------- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/core/lib/iomgr/endpoint_pair_windows.c b/src/core/lib/iomgr/endpoint_pair_windows.c index dcf8166104e..25d6264dfb6 100644 --- a/src/core/lib/iomgr/endpoint_pair_windows.c +++ b/src/core/lib/iomgr/endpoint_pair_windows.c @@ -88,10 +88,12 @@ grpc_endpoint_pair grpc_iomgr_create_endpoint_pair( grpc_endpoint_pair p; create_sockets(sv); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - p.client = grpc_tcp_create(&exec_ctx, grpc_winsocket_create(sv[1], "endpoint:client"), + p.client = grpc_tcp_create(&exec_ctx, + grpc_winsocket_create(sv[1], "endpoint:client"), channel_args, "endpoint:server"); - p.server = grpc_tcp_create(&exec_ctx,grpc_winsocket_create(sv[0], "endpoint:server"), - channel_args, "endpoint:client"); + p.server = grpc_tcp_create(&exec_ctx, + grpc_winsocket_create(sv[0], "endpoint:server"), + channel_args, "endpoint:client"); grpc_exec_ctx_finish(&exec_ctx); return p; } diff --git a/src/core/lib/iomgr/tcp_client_windows.c b/src/core/lib/iomgr/tcp_client_windows.c index 935f96b810e..d6baca50baf 100644 --- a/src/core/lib/iomgr/tcp_client_windows.c +++ b/src/core/lib/iomgr/tcp_client_windows.c @@ -72,7 +72,7 @@ static void async_connect_unlock_and_cleanup(grpc_exec_ctx *exec_ctx, int done = (--ac->refs == 0); gpr_mu_unlock(&ac->mu); if (done) { - grpc_channel_args_destroy(exec_ctx, ac->channel_args); + grpc_channel_args_destroy(exec_ctx, ac->channel_args); gpr_mu_destroy(&ac->mu); gpr_free(ac->addr_name); gpr_free(ac); @@ -119,7 +119,8 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) { if (!wsa_success) { error = GRPC_WSA_ERROR(WSAGetLastError(), "ConnectEx"); } else { - *ep = grpc_tcp_create(exec_ctx, socket, ac->channel_args, ac->addr_name); + *ep = + grpc_tcp_create(exec_ctx, socket, ac->channel_args, ac->addr_name); socket = NULL; } } else { diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c index 3b077e6d1ff..4c17f08918b 100644 --- a/src/core/lib/iomgr/tcp_server_windows.c +++ b/src/core/lib/iomgr/tcp_server_windows.c @@ -46,6 +46,7 @@ #include #include +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/iocp_windows.h" #include "src/core/lib/iomgr/pollset_windows.h" #include "src/core/lib/iomgr/resolve_address.h" @@ -53,7 +54,6 @@ #include "src/core/lib/iomgr/socket_windows.h" #include "src/core/lib/iomgr/tcp_server.h" #include "src/core/lib/iomgr/tcp_windows.h" -#include "src/core/lib/channel/channel_args.h" #define MIN_SAFE_ACCEPT_QUEUE_SIZE 100 diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c index 35b991cb127..f74aa687936 100644 --- a/src/core/lib/iomgr/tcp_windows.c +++ b/src/core/lib/iomgr/tcp_windows.c @@ -433,18 +433,17 @@ static grpc_endpoint_vtable vtable = {win_read, grpc_endpoint *grpc_tcp_create(grpc_exec_ctx *exec_ctx, grpc_winsocket *socket, grpc_channel_args *channel_args, char *peer_string) { - grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); - if (channel_args != NULL) { - for (size_t i = 0; i < channel_args->num_args; i++) { -if (0 == - strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { - grpc_resource_quota_unref_internal(exec_ctx, resource_quota); - resource_quota = grpc_resource_quota_ref_internal( - channel_args->args[i].value.pointer.p); - } - } - } - grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); + grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL); + if (channel_args != NULL) { + for (size_t i = 0; i < channel_args->num_args; i++) { + if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) { + grpc_resource_quota_unref_internal(exec_ctx, resource_quota); + resource_quota = grpc_resource_quota_ref_internal( + channel_args->args[i].value.pointer.p); + } + } + } + grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp)); memset(tcp, 0, sizeof(grpc_tcp)); tcp->base.vtable = &vtable; tcp->socket = socket; From d4da50b2cb1575b9a2661c47e7d393931d28bc84 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Wed, 5 Apr 2017 10:50:03 -0700 Subject: [PATCH 80/90] Add manifest to fix grpcio_reflection packaging --- src/python/grpcio_reflection/MANIFEST.in | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/python/grpcio_reflection/MANIFEST.in diff --git a/src/python/grpcio_reflection/MANIFEST.in b/src/python/grpcio_reflection/MANIFEST.in new file mode 100644 index 00000000000..0f2130c0b58 --- /dev/null +++ b/src/python/grpcio_reflection/MANIFEST.in @@ -0,0 +1,4 @@ +include grpc_version.py +include reflection_commands.py +graft grpc_reflection +global-exclude *.pyc From 5fa46127b67351bcd00fc6eefdb9846366ba483c Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Wed, 5 Apr 2017 15:41:18 -0700 Subject: [PATCH 81/90] Fix headers included in max_age_filter.c --- src/core/ext/filters/max_age/max_age_filter.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/ext/filters/max_age/max_age_filter.c b/src/core/ext/filters/max_age/max_age_filter.c index f858220c010..a045f0a421a 100644 --- a/src/core/ext/filters/max_age/max_age_filter.c +++ b/src/core/ext/filters/max_age/max_age_filter.c @@ -31,7 +31,7 @@ * */ -#include "src/core/lib/channel/message_size_filter.h" +#include "src/core/ext/filters/max_age/max_age_filter.h" #include #include @@ -41,7 +41,6 @@ #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/surface/channel_init.h" #include "src/core/lib/transport/http2_errors.h" -#include "src/core/lib/transport/service_config.h" #define DEFAULT_MAX_CONNECTION_AGE_MS INT_MAX #define DEFAULT_MAX_CONNECTION_AGE_GRACE_MS INT_MAX From 9fd9a442abf5fdd5c04d19d6d8a2ff1ef3b9e5d7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 16:52:34 -0700 Subject: [PATCH 82/90] Fix nullptr deref --- src/core/lib/surface/call.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 315c906baae..97d50a91be5 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -529,14 +529,14 @@ static void destroy_call(grpc_exec_ctx *exec_ctx, void *call, } void grpc_call_destroy(grpc_call *c) { - parent_call *pc = get_parent_call(c); child_call *cc = c->child_call; grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; GPR_TIMER_BEGIN("grpc_call_destroy", 0); GRPC_API_TRACE("grpc_call_destroy(c=%p)", 1, (c)); - if (pc) { + if (cc) { + parent_call *pc = get_parent_call(cc->parent); gpr_mu_lock(&pc->child_list_mu); if (c == pc->first_child) { pc->first_child = cc->sibling_next; From 557c88c3ac835f343b6510f5b6ed309773042c87 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 17:20:18 -0700 Subject: [PATCH 83/90] Change pollset rules --- src/core/lib/iomgr/ev_epoll_linux.c | 4 ++-- src/core/lib/iomgr/ev_poll_posix.c | 4 ++-- src/core/lib/iomgr/pollset.h | 4 ++++ src/core/lib/iomgr/pollset_windows.c | 4 ++-- src/core/lib/surface/completion_queue.c | 5 ++--- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 7014b983495..e5cf54f10a9 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -1717,7 +1717,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, worker.pt_id = pthread_self(); gpr_atm_no_barrier_store(&worker.is_kicked, (gpr_atm)0); - *worker_hdl = &worker; + if (worker_hdl) *worker_hdl = &worker; gpr_tls_set(&g_current_thread_pollset, (intptr_t)pollset); gpr_tls_set(&g_current_thread_worker, (intptr_t)&worker); @@ -1795,7 +1795,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_mu_lock(&pollset->po.mu); } - *worker_hdl = NULL; + if (worker_hdl) *worker_hdl = NULL; gpr_tls_set(&g_current_thread_pollset, (intptr_t)0); gpr_tls_set(&g_current_thread_worker, (intptr_t)0); diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c index d90f2233622..9834cdd1979 100644 --- a/src/core/lib/iomgr/ev_poll_posix.c +++ b/src/core/lib/iomgr/ev_poll_posix.c @@ -871,7 +871,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - *worker_hdl = &worker; + if (worker_hdl) *worker_hdl = &worker; grpc_error *error = GRPC_ERROR_NONE; /* Avoid malloc for small number of elements. */ @@ -1092,7 +1092,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_mu_lock(&pollset->mu); } } - *worker_hdl = NULL; + if (worker_hdl) *worker_hdl = NULL; GPR_TIMER_END("pollset_work", 0); GRPC_LOG_IF_ERROR("pollset_work", GRPC_ERROR_REF(error)); return error; diff --git a/src/core/lib/iomgr/pollset.h b/src/core/lib/iomgr/pollset.h index e19ce697b8d..9bf3cdac89e 100644 --- a/src/core/lib/iomgr/pollset.h +++ b/src/core/lib/iomgr/pollset.h @@ -75,6 +75,10 @@ void grpc_pollset_destroy(grpc_pollset *pollset); and it is guaranteed that it will not be released by grpc_pollset_work AFTER worker has been destroyed. + It's legal for worker to be NULL: in that case, this specific thread can not + be directly woken with a kick, but maybe be indirectly (with a kick against + the pollset as a whole). + Tries not to block past deadline. May call grpc_closure_list_run on grpc_closure_list, without holding the pollset diff --git a/src/core/lib/iomgr/pollset_windows.c b/src/core/lib/iomgr/pollset_windows.c index 17043c1ea1a..04c6b717475 100644 --- a/src/core/lib/iomgr/pollset_windows.c +++ b/src/core/lib/iomgr/pollset_windows.c @@ -120,7 +120,7 @@ grpc_error *grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker **worker_hdl, gpr_timespec now, gpr_timespec deadline) { grpc_pollset_worker worker; - *worker_hdl = &worker; + if (worker_hdl) *worker_hdl = &worker; int added_worker = 0; worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = @@ -185,7 +185,7 @@ done: remove_worker(&worker, GRPC_POLLSET_WORKER_LINK_POLLSET); } gpr_cv_destroy(&worker.cv); - *worker_hdl = NULL; + if (worker_hdl) *worker_hdl = NULL; return GRPC_ERROR_NONE; } diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c index b4594817e45..3273addf1d0 100644 --- a/src/core/lib/surface/completion_queue.c +++ b/src/core/lib/surface/completion_queue.c @@ -345,7 +345,6 @@ static void dump_pending_tags(grpc_completion_queue *cc) {} grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_timespec deadline, void *reserved) { grpc_event ret; - grpc_pollset_worker *worker = NULL; gpr_timespec now; GPR_TIMER_BEGIN("grpc_completion_queue_next", 0); @@ -426,8 +425,8 @@ grpc_event grpc_completion_queue_next(grpc_completion_queue *cc, gpr_mu_lock(cc->mu); continue; } else { - grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), - &worker, now, iteration_deadline); + grpc_error *err = grpc_pollset_work(&exec_ctx, POLLSET_FROM_CQ(cc), NULL, + now, iteration_deadline); if (err != GRPC_ERROR_NONE) { gpr_mu_unlock(cc->mu); const char *msg = grpc_error_string(err); From dd2f1cab7a091f90070973a20e6693c13fa30870 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 6 Apr 2017 13:00:26 +0200 Subject: [PATCH 84/90] switch C# distribtest to use windows-build nugets --- test/distrib/csharp/run_distrib_test.bat | 2 +- test/distrib/csharp/run_distrib_test.sh | 2 +- test/distrib/csharp/run_distrib_test_dotnetcli.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/distrib/csharp/run_distrib_test.bat b/test/distrib/csharp/run_distrib_test.bat index 6cf381142f4..cb5dd552739 100644 --- a/test/distrib/csharp/run_distrib_test.bat +++ b/test/distrib/csharp/run_distrib_test.bat @@ -31,7 +31,7 @@ cd /d %~dp0 @rem extract input artifacts -powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('../../../input_artifacts/csharp_nugets_dotnetcli.zip', 'TestNugetFeed');" +powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::ExtractToDirectory('../../../input_artifacts/csharp_nugets_windows_dotnetcli.zip', 'TestNugetFeed');" update_version.sh auto diff --git a/test/distrib/csharp/run_distrib_test.sh b/test/distrib/csharp/run_distrib_test.sh index 0a77c1af440..9de5ce0cd32 100755 --- a/test/distrib/csharp/run_distrib_test.sh +++ b/test/distrib/csharp/run_distrib_test.sh @@ -32,7 +32,7 @@ set -ex cd $(dirname $0) -unzip -o "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets_dotnetcli.zip" -d TestNugetFeed +unzip -o "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets_windows_dotnetcli.zip" -d TestNugetFeed ./update_version.sh auto diff --git a/test/distrib/csharp/run_distrib_test_dotnetcli.sh b/test/distrib/csharp/run_distrib_test_dotnetcli.sh index 493c5049fb8..cdfc91bf427 100755 --- a/test/distrib/csharp/run_distrib_test_dotnetcli.sh +++ b/test/distrib/csharp/run_distrib_test_dotnetcli.sh @@ -32,7 +32,7 @@ set -ex cd $(dirname $0) -unzip -o "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets_dotnetcli.zip" -d TestNugetFeed +unzip -o "$EXTERNAL_GIT_ROOT/input_artifacts/csharp_nugets_windows_dotnetcli.zip" -d TestNugetFeed ./update_version.sh auto From c4c95ada896c28931869275fc1a7cff62180e12b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 6 Apr 2017 13:20:49 +0200 Subject: [PATCH 85/90] make C# docker image in sync with GCE performance workers --- src/csharp/global.json | 2 +- templates/tools/dockerfile/csharp_dotnetcli_deps.include | 4 ++-- .../interoptest/grpc_interop_csharpcoreclr/Dockerfile | 4 ++-- tools/dockerfile/test/csharp_coreclr_x64/Dockerfile | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/csharp/global.json b/src/csharp/global.json index 32ff399ef94..f3c33cef6a5 100644 --- a/src/csharp/global.json +++ b/src/csharp/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "1.0.0-preview2-003121" + "version": "1.0.0-preview2-003131" } } \ No newline at end of file diff --git a/templates/tools/dockerfile/csharp_dotnetcli_deps.include b/templates/tools/dockerfile/csharp_dotnetcli_deps.include index 058ce15fb52..bc88d2bfa39 100644 --- a/templates/tools/dockerfile/csharp_dotnetcli_deps.include +++ b/templates/tools/dockerfile/csharp_dotnetcli_deps.include @@ -1,7 +1,7 @@ # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext -# dotnet-dev-1.0.0-preview2-003121 -RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 +# dotnet-dev-1.0.0-preview2-003131 +RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=827530 RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet # dotnet-dev-1.0.1 RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index c26c9a2826e..2a59628b487 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -99,8 +99,8 @@ RUN nuget update -self # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext -# dotnet-dev-1.0.0-preview2-003121 -RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 +# dotnet-dev-1.0.0-preview2-003131 +RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=827530 RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet # dotnet-dev-1.0.1 RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 diff --git a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile index c26c9a2826e..2a59628b487 100644 --- a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile @@ -99,8 +99,8 @@ RUN nuget update -self # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext -# dotnet-dev-1.0.0-preview2-003121 -RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 +# dotnet-dev-1.0.0-preview2-003131 +RUN curl -sSL -o dotnet100.tar.gz https://go.microsoft.com/fwlink/?LinkID=827530 RUN mkdir -p /opt/dotnet && tar zxf dotnet100.tar.gz -C /opt/dotnet # dotnet-dev-1.0.1 RUN curl -sSL -o dotnet101.tar.gz https://go.microsoft.com/fwlink/?LinkID=843453 From 631e78c34975f7d1cb030cd8d1b5593ed86a2e89 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Thu, 6 Apr 2017 10:44:45 -0700 Subject: [PATCH 86/90] Print items/sec metric at the end --- test/cpp/microbenchmarks/bm_cq_multiple_threads.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index 80f21f1f697..967c226ac73 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -124,6 +124,8 @@ static void BM_Cq_Throughput(benchmark::State& state) { grpc_completion_queue_next(g_cq, deadline, NULL); } + state.SetItemsProcessed(state.iterations()); + if (state.thread_index == 0) { grpc_completion_queue_shutdown(g_cq); grpc_completion_queue_destroy(g_cq); From c6ec1155d026c91b1badb07ef1605bb747cff064 Mon Sep 17 00:00:00 2001 From: Noah Eisen Date: Wed, 5 Apr 2017 16:10:13 -0700 Subject: [PATCH 87/90] Fix error overflow bug --- src/core/lib/iomgr/error.c | 35 ++++++++++++++++-- .../clusterfuzz-testcase-6312731374256128 | Bin 0 -> 26793 bytes test/core/iomgr/error_test.c | 30 ++++++++++++++- tools/run_tests/generated/tests.json | 23 ++++++++++++ 4 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 test/core/end2end/fuzzers/server_fuzzer_corpus/clusterfuzz-testcase-6312731374256128 diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 1dbb64e8f37..766cc8a5169 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -212,7 +212,11 @@ static uint8_t get_placement(grpc_error **err, size_t size) { GPR_ASSERT(*err); uint8_t slots = (uint8_t)(size / sizeof(intptr_t)); if ((*err)->arena_size + slots > (*err)->arena_capacity) { - (*err)->arena_capacity = (uint8_t)(3 * (*err)->arena_capacity / 2); + (*err)->arena_capacity = + (uint8_t)GPR_MIN(UINT8_MAX - 1, (3 * (*err)->arena_capacity / 2)); + if ((*err)->arena_size + slots > (*err)->arena_capacity) { + return UINT8_MAX; + } *err = gpr_realloc( *err, sizeof(grpc_error) + (*err)->arena_capacity * sizeof(intptr_t)); } @@ -223,10 +227,14 @@ static uint8_t get_placement(grpc_error **err, size_t size) { static void internal_set_int(grpc_error **err, grpc_error_ints which, intptr_t value) { - // GPR_ASSERT((*err)->ints[which] == UINT8_MAX); // TODO, enforce this uint8_t slot = (*err)->ints[which]; if (slot == UINT8_MAX) { slot = get_placement(err, sizeof(value)); + if (slot == UINT8_MAX) { + gpr_log(GPR_ERROR, "Error %p is full, dropping int {\"%s\":%" PRIiPTR "}", *err, + error_int_name(which), value); + return; + } } (*err)->ints[which] = slot; (*err)->arena[slot] = value; @@ -234,10 +242,16 @@ static void internal_set_int(grpc_error **err, grpc_error_ints which, static void internal_set_str(grpc_error **err, grpc_error_strs which, grpc_slice value) { - // GPR_ASSERT((*err)->strs[which] == UINT8_MAX); // TODO, enforce this uint8_t slot = (*err)->strs[which]; if (slot == UINT8_MAX) { slot = get_placement(err, sizeof(value)); + if (slot == UINT8_MAX) { + const char *str = grpc_slice_to_c_string(value); + gpr_log(GPR_ERROR, "Error %p is full, dropping string {\"%s\":\"%s\"}", + *err, error_str_name(which), str); + gpr_free((void *)str); + return; + } } else { unref_slice(*(grpc_slice *)((*err)->arena + slot)); } @@ -245,12 +259,19 @@ static void internal_set_str(grpc_error **err, grpc_error_strs which, memcpy((*err)->arena + slot, &value, sizeof(value)); } +static char *fmt_time(gpr_timespec tm); static void internal_set_time(grpc_error **err, grpc_error_times which, gpr_timespec value) { - // GPR_ASSERT((*err)->times[which] == UINT8_MAX); // TODO, enforce this uint8_t slot = (*err)->times[which]; if (slot == UINT8_MAX) { slot = get_placement(err, sizeof(value)); + if (slot == UINT8_MAX) { + const char *time_str = fmt_time(value); + gpr_log(GPR_ERROR, "Error %p is full, dropping \"%s\":\"%s\"}", *err, + error_time_name(which), time_str); + gpr_free((void *)time_str); + return; + } } (*err)->times[which] = slot; memcpy((*err)->arena + slot, &value, sizeof(value)); @@ -259,6 +280,12 @@ static void internal_set_time(grpc_error **err, grpc_error_times which, static void internal_add_error(grpc_error **err, grpc_error *new) { grpc_linked_error new_last = {new, UINT8_MAX}; uint8_t slot = get_placement(err, sizeof(grpc_linked_error)); + if (slot == UINT8_MAX) { + gpr_log(GPR_ERROR, "Error %p is full, dropping error %p = %s", *err, new, + grpc_error_string(new)); + GRPC_ERROR_UNREF(new); + return; + } if ((*err)->first_err == UINT8_MAX) { GPR_ASSERT((*err)->last_err == UINT8_MAX); (*err)->last_err = slot; diff --git a/test/core/end2end/fuzzers/server_fuzzer_corpus/clusterfuzz-testcase-6312731374256128 b/test/core/end2end/fuzzers/server_fuzzer_corpus/clusterfuzz-testcase-6312731374256128 new file mode 100644 index 0000000000000000000000000000000000000000..4c6eb601ae342400eaccb7d4688136d152f2b06d GIT binary patch literal 26793 zcmeI*u}Z@L6h`3)rAV>w(Ah!7r_ezt+V_8zn%nSuFE|ShUxyeYT@L>_2@dV~^ZoRA zdb?cC&#zA}$HVdP^-;%V+3%LyYyEM(u4lWu?e|=-YwM-^{dW21n7@1950cvnF80Es zc7hLl-~%7h0nZbjv)TsFpM2m0AG!m4-~%7{kmp7JUz#@bE^xu62*eZjP4~@dM0TMQ zlS=!73ob<Z4LUFgK5(!St=OA&}C`l+AOi0ncq zCYAOD7hH-!Jkd}6oJM39Ix(rVFSy`R1mcN)>gO~fyU>YArG3E#mm&~P^iw~l5!r=K zOe*aQF1Qqdc%q;BIgQ9JbYfCzUvR;t2*eZp)X!-|cA*oKO8bHfE=3@o=%;>8BeDyf zm{i&qTyQA@@kBrMa~hFd=)|PbzTkpO5r`-Hsh`t`>_R6dmG%V}T#7(E(NF!HMr0Q{ zF{!jKxZqL*;)#Ch=QJX_(1}T Date: Thu, 6 Apr 2017 10:53:57 -0700 Subject: [PATCH 88/90] Correct use of ProtoBuf.js 6 message encoding API --- src/node/src/protobuf_js_6_common.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/node/src/protobuf_js_6_common.js b/src/node/src/protobuf_js_6_common.js index 21eddabaa8a..00f11f27363 100644 --- a/src/node/src/protobuf_js_6_common.js +++ b/src/node/src/protobuf_js_6_common.js @@ -77,11 +77,7 @@ exports.serializeCls = function serializeCls(cls) { * @return {Buffer} The serialized object */ return function serialize(arg) { - var errMsg = cls.verify(arg); - if (errMsg) { - throw Error(errMsg); - } - var message = cls.create(arg); + var message = cls.fromObject(arg); return cls.encode(message).finish(); }; }; From fc8d671ec8b2ad977d2df6c712e2a08ab13e16ca Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 6 Apr 2017 11:39:21 -0700 Subject: [PATCH 89/90] Fix posix leakage --- include/grpc/impl/codegen/grpc_types.h | 8 ++++++++ src/core/lib/iomgr/tcp_posix.h | 10 ---------- test/core/end2end/fixtures/h2_sockpair_1byte.c | 1 - test/core/iomgr/endpoint_pair_test.c | 1 - test/core/security/secure_endpoint_test.c | 1 - 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 8a8b20650fa..fdedf0a84f3 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -271,6 +271,14 @@ typedef struct { * possible. */ #define GRPC_ARG_USE_CRONET_PACKET_COALESCING \ "grpc.use_cronet_packet_coalescing" +/* Channel arg (integer) setting how large a slice to try and read from the wire +each time recvmsg (or equivalent) is called */ +#define GRPC_ARG_TCP_READ_CHUNK_SIZE "grpc.experimental.tcp_read_chunk_size" +#define GRPC_TCP_DEFAULT_READ_SLICE_SIZE 8192 +#define GRPC_ARG_TCP_MIN_READ_CHUNK_SIZE \ + "grpc.experimental.tcp_min_read_chunk_size" +#define GRPC_ARG_TCP_MAX_READ_CHUNK_SIZE \ + "grpc.experimental.tcp_max_read_chunk_size" /** \} */ /** Result of a grpc call. If the caller satisfies the prerequisites of a diff --git a/src/core/lib/iomgr/tcp_posix.h b/src/core/lib/iomgr/tcp_posix.h index 3bf93dfcf85..1ad5788331f 100644 --- a/src/core/lib/iomgr/tcp_posix.h +++ b/src/core/lib/iomgr/tcp_posix.h @@ -47,16 +47,6 @@ #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/ev_posix.h" -#define GRPC_TCP_DEFAULT_READ_SLICE_SIZE 8192 - -/* Channel arg (integer) setting how large a slice to try and read from the wire - each time recvmsg (or equivalent) is called */ -#define GRPC_ARG_TCP_READ_CHUNK_SIZE "grpc.experimental.tcp_read_chunk_size" -#define GRPC_ARG_TCP_MIN_READ_CHUNK_SIZE \ - "grpc.experimental.tcp_min_read_chunk_size" -#define GRPC_ARG_TCP_MAX_READ_CHUNK_SIZE \ - "grpc.experimental.tcp_max_read_chunk_size" - extern int grpc_tcp_trace; /* Create a tcp endpoint given a file desciptor and a read slice size. diff --git a/test/core/end2end/fixtures/h2_sockpair_1byte.c b/test/core/end2end/fixtures/h2_sockpair_1byte.c index 2b87602adb6..04174fa5015 100644 --- a/test/core/end2end/fixtures/h2_sockpair_1byte.c +++ b/test/core/end2end/fixtures/h2_sockpair_1byte.c @@ -48,7 +48,6 @@ #include "src/core/lib/channel/http_server_filter.h" #include "src/core/lib/iomgr/endpoint_pair.h" #include "src/core/lib/iomgr/iomgr.h" -#include "src/core/lib/iomgr/tcp_posix.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/surface/completion_queue.h" #include "src/core/lib/surface/server.h" diff --git a/test/core/iomgr/endpoint_pair_test.c b/test/core/iomgr/endpoint_pair_test.c index 491b05f41fd..c8a60776b9c 100644 --- a/test/core/iomgr/endpoint_pair_test.c +++ b/test/core/iomgr/endpoint_pair_test.c @@ -37,7 +37,6 @@ #include #include #include -#include "src/core/lib/iomgr/tcp_posix.h" #include "test/core/iomgr/endpoint_tests.h" #include "test/core/util/test_config.h" diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c index f6691f8adcd..71d8057ac31 100644 --- a/test/core/security/secure_endpoint_test.c +++ b/test/core/security/secure_endpoint_test.c @@ -42,7 +42,6 @@ #include #include "src/core/lib/iomgr/endpoint_pair.h" #include "src/core/lib/iomgr/iomgr.h" -#include "src/core/lib/iomgr/tcp_posix.h" #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/tsi/fake_transport_security.h" From 786c852035aac17ff8584a900c6206cc151062e8 Mon Sep 17 00:00:00 2001 From: Noah Eisen Date: Thu, 6 Apr 2017 15:48:19 -0700 Subject: [PATCH 90/90] clang fmt If I had a nickle... --- src/core/lib/iomgr/error.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/error.c b/src/core/lib/iomgr/error.c index 766cc8a5169..fbbca6b4939 100644 --- a/src/core/lib/iomgr/error.c +++ b/src/core/lib/iomgr/error.c @@ -231,8 +231,8 @@ static void internal_set_int(grpc_error **err, grpc_error_ints which, if (slot == UINT8_MAX) { slot = get_placement(err, sizeof(value)); if (slot == UINT8_MAX) { - gpr_log(GPR_ERROR, "Error %p is full, dropping int {\"%s\":%" PRIiPTR "}", *err, - error_int_name(which), value); + gpr_log(GPR_ERROR, "Error %p is full, dropping int {\"%s\":%" PRIiPTR "}", + *err, error_int_name(which), value); return; } }