From 32baa5e62209b67ff247e4a03803c70cb364b457 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 27 Jan 2016 10:01:41 -0800 Subject: [PATCH 01/11] Allow dynamic sizing of async client and server thread pools --- test/cpp/qps/client_async.cc | 62 ++++++++++++++++++++++++------------ test/cpp/qps/server_async.cc | 38 ++++++++++++++-------- 2 files changed, 65 insertions(+), 35 deletions(-) diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index f270cd09875..13e71fec7d9 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -46,13 +46,14 @@ #include #include #include +#include #include #include +#include "src/proto/grpc/testing/services.grpc.pb.h" #include "test/cpp/qps/client.h" #include "test/cpp/qps/timer.h" #include "test/cpp/util/create_test_channel.h" -#include "src/proto/grpc/testing/services.grpc.pb.h" namespace grpc { namespace testing { @@ -93,7 +94,8 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function< std::unique_ptr>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*)> start_req, + CompletionQueue*)> + start_req, std::function on_done) : ClientRpcContext(channel_id), context_(), @@ -139,7 +141,8 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function callback_; std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*)> start_req_; + CompletionQueue*)> + start_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -158,20 +161,22 @@ class AsyncClient : public ClientImpl { using Client::closed_loop_; using ClientImpl::channels_; using ClientImpl::request_; - AsyncClient(const ClientConfig& config, - std::function setup_ctx, - std::function(std::shared_ptr)> - create_stub) + AsyncClient( + const ClientConfig& config, + std::function + setup_ctx, + std::function(std::shared_ptr)> + create_stub) : ClientImpl(config, create_stub), + num_async_threads_(NumThreads(config)), channel_lock_(new std::mutex[config.client_channels()]), contexts_(config.client_channels()), max_outstanding_per_channel_(config.outstanding_rpcs_per_channel()), channel_count_(config.client_channels()), - pref_channel_inc_(config.async_client_threads()) { - SetupLoadTest(config, config.async_client_threads()); + pref_channel_inc_(num_async_threads_) { + SetupLoadTest(config, num_async_threads_); - for (int i = 0; i < config.async_client_threads(); i++) { + for (int i = 0; i < num_async_threads_; i++) { cli_cqs_.emplace_back(new CompletionQueue); if (!closed_loop_) { rpc_deadlines_.emplace_back(); @@ -324,6 +329,9 @@ class AsyncClient : public ClientImpl { return true; } + protected: + int num_async_threads_; + private: class boolean { // exists only to avoid data-race on vector public: @@ -338,6 +346,15 @@ class AsyncClient : public ClientImpl { private: bool val_; }; + static int NumThreads(const ClientConfig& config) { + int num_threads = config.async_client_threads(); + if (num_threads <= 0) { // Use dynamic sizing + num_threads = gpr_cpu_num_cores(); + gpr_log(GPR_INFO, "Sizing client server to %d threads\n", num_threads); + } + return num_threads; + } + std::vector> cli_cqs_; std::vector rpc_deadlines_; // per thread deadlines @@ -363,7 +380,7 @@ class AsyncUnaryClient GRPC_FINAL public: explicit AsyncUnaryClient(const ClientConfig& config) : AsyncClient(config, SetupCtx, BenchmarkStubCreator) { - StartThreads(config.async_client_threads()); + StartThreads(num_async_threads_); } ~AsyncUnaryClient() GRPC_OVERRIDE { EndThreads(); } @@ -391,7 +408,8 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { std::function>( BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, - void*)> start_req, + void*)> + start_req, std::function on_done) : ClientRpcContext(channel_id), context_(), @@ -443,10 +461,10 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { ResponseType response_; bool (ClientRpcContextStreamingImpl::*next_state_)(bool, Histogram*); std::function callback_; - std::function< - std::unique_ptr>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, - void*)> start_req_; + std::function>( + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, void*)> + start_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -461,7 +479,7 @@ class AsyncStreamingClient GRPC_FINAL // async streaming currently only supports closed loop GPR_ASSERT(closed_loop_); - StartThreads(config.async_client_threads()); + StartThreads(num_async_threads_); } ~AsyncStreamingClient() GRPC_OVERRIDE { EndThreads(); } @@ -490,7 +508,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { int channel_id, grpc::GenericStub* stub, const ByteBuffer& req, std::function( grpc::GenericStub*, grpc::ClientContext*, - const grpc::string& method_name, CompletionQueue*, void*)> start_req, + const grpc::string& method_name, CompletionQueue*, void*)> + start_req, std::function on_done) : ClientRpcContext(channel_id), context_(), @@ -547,7 +566,8 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function callback_; std::function( grpc::GenericStub*, grpc::ClientContext*, const grpc::string&, - CompletionQueue*, void*)> start_req_; + CompletionQueue*, void*)> + start_req_; grpc::Status status_; double start_; std::unique_ptr stream_; @@ -566,7 +586,7 @@ class GenericAsyncStreamingClient GRPC_FINAL // async streaming currently only supports closed loop GPR_ASSERT(closed_loop_); - StartThreads(config.async_client_threads()); + StartThreads(num_async_threads_); } ~GenericAsyncStreamingClient() GRPC_OVERRIDE { EndThreads(); } diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index cd8d546c283..8e1503b4604 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -50,8 +50,8 @@ #include #include -#include "test/cpp/qps/server.h" #include "src/proto/grpc/testing/services.grpc.pb.h" +#include "test/cpp/qps/server.h" namespace grpc { namespace testing { @@ -72,7 +72,8 @@ class AsyncQpsServerTest : public Server { CompletionQueue *, ServerCompletionQueue *, void *)> request_streaming_function, std::function process_rpc) + ResponseType *)> + process_rpc) : Server(config) { char *server_address = NULL; @@ -85,7 +86,13 @@ class AsyncQpsServerTest : public Server { register_service(&builder, &async_service_); - for (int i = 0; i < config.async_server_threads(); i++) { + int num_threads = config.async_server_threads(); + if (num_threads <= 0) { // dynamic sizing + num_threads = cores(); + gpr_log(GPR_INFO, "Sizing async server to %d threads\n", num_threads); + } + + for (int i = 0; i < num_threads; i++) { srv_cqs_.emplace_back(builder.AddCompletionQueue()); } @@ -96,8 +103,8 @@ class AsyncQpsServerTest : public Server { auto process_rpc_bound = std::bind(process_rpc, config.payload_config(), _1, _2); - for (int i = 0; i < 10000 / config.async_server_threads(); i++) { - for (int j = 0; j < config.async_server_threads(); j++) { + for (int i = 0; i < 10000 / num_threads; i++) { + for (int j = 0; j < num_threads; j++) { if (request_unary_function) { auto request_unary = std::bind(request_unary_function, &async_service_, _1, _2, _3, @@ -115,10 +122,10 @@ class AsyncQpsServerTest : public Server { } } - for (int i = 0; i < config.async_server_threads(); i++) { + for (int i = 0; i < num_threads; i++) { shutdown_state_.emplace_back(new PerThreadShutdownState()); } - for (int i = 0; i < config.async_server_threads(); i++) { + for (int i = 0; i < num_threads; i++) { threads_.emplace_back(&AsyncQpsServerTest::ThreadFunc, this, i); } } @@ -184,7 +191,8 @@ class AsyncQpsServerTest : public Server { ServerRpcContextUnaryImpl( std::function *, - void *)> request_method, + void *)> + request_method, std::function invoke_method) : srv_ctx_(new ServerContextType), @@ -381,12 +389,14 @@ static Status ProcessGenericRPC(const PayloadConfig &payload_config, } std::unique_ptr CreateAsyncServer(const ServerConfig &config) { - return std::unique_ptr(new AsyncQpsServerTest< - SimpleRequest, SimpleResponse, BenchmarkService::AsyncService, - grpc::ServerContext>( - config, RegisterBenchmarkService, - &BenchmarkService::AsyncService::RequestUnaryCall, - &BenchmarkService::AsyncService::RequestStreamingCall, ProcessSimpleRPC)); + return std::unique_ptr( + new AsyncQpsServerTest( + config, RegisterBenchmarkService, + &BenchmarkService::AsyncService::RequestUnaryCall, + &BenchmarkService::AsyncService::RequestStreamingCall, + ProcessSimpleRPC)); } std::unique_ptr CreateAsyncGenericServer(const ServerConfig &config) { return std::unique_ptr( From d02dd30a1fb0895bac9f6139bf8ef9418a26b69d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 27 Jan 2016 11:23:34 -0800 Subject: [PATCH 02/11] Also dynamic sizing for num_clients on performance tests --- test/cpp/qps/driver.cc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index acb265b3086..4ef70a25700 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -31,24 +31,24 @@ * */ +#include #include #include -#include #include -#include -#include -#include #include #include +#include +#include +#include #include "src/core/support/env.h" +#include "src/proto/grpc/testing/services.grpc.pb.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" #include "test/cpp/qps/driver.h" #include "test/cpp/qps/histogram.h" #include "test/cpp/qps/qps_worker.h" -#include "src/proto/grpc/testing/services.grpc.pb.h" using std::list; using std::thread; @@ -142,6 +142,12 @@ std::unique_ptr RunScenario( } } + // if num_clients is set to <=0, do dynamic sizing: all workers + // except for servers are clients + if (num_clients <= 0) { + num_clients = workers.size() - num_servers; + } + // TODO(ctiller): support running multiple configurations, and binpack // client/server pairs // to available workers From 7a984f0e3aa65367dac193d8f616bd7692e61998 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 27 Jan 2016 11:24:57 -0800 Subject: [PATCH 03/11] Copyright --- test/cpp/qps/driver.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/qps/driver.cc b/test/cpp/qps/driver.cc index 4ef70a25700..240c66d7a3b 100644 --- a/test/cpp/qps/driver.cc +++ b/test/cpp/qps/driver.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without From 7d8335f8753afe33117358bee32f89808eae5d31 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 27 Jan 2016 12:00:41 -0800 Subject: [PATCH 04/11] Don't log newlines --- test/cpp/qps/client_async.cc | 2 +- test/cpp/qps/server_async.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 13e71fec7d9..4ab74a9d4c6 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -350,7 +350,7 @@ class AsyncClient : public ClientImpl { int num_threads = config.async_client_threads(); if (num_threads <= 0) { // Use dynamic sizing num_threads = gpr_cpu_num_cores(); - gpr_log(GPR_INFO, "Sizing client server to %d threads\n", num_threads); + gpr_log(GPR_INFO, "Sizing client server to %d threads", num_threads); } return num_threads; } diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index 8e1503b4604..ddfeb8638d5 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -89,7 +89,7 @@ class AsyncQpsServerTest : public Server { int num_threads = config.async_server_threads(); if (num_threads <= 0) { // dynamic sizing num_threads = cores(); - gpr_log(GPR_INFO, "Sizing async server to %d threads\n", num_threads); + gpr_log(GPR_INFO, "Sizing async server to %d threads", num_threads); } for (int i = 0; i < num_threads; i++) { From 788ee23ab3171da1db0aa9c6c250e0fd2eaaebc8 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 26 Jan 2016 12:19:44 -0800 Subject: [PATCH 05/11] cleanup sanity tests --- .../grpc_jenkins_slave_x64/Dockerfile | 2 - .../grpc_jenkins_slave_x86/Dockerfile | 2 - tools/dockerfile/grpc_sanity/Dockerfile | 92 +++++++++++++++++++ tools/jenkins/docker_run_tests.sh | 10 +- tools/run_tests/run_tests.py | 59 +++++++++--- .../run_tests/{ => sanity}/check_cache_mk.sh | 0 .../{ => sanity}/check_sources_and_headers.py | 66 ++++++------- .../{ => sanity}/check_submodules.sh | 2 +- .../run_tests/{ => sanity}/sanity_tests.yaml | 6 +- 9 files changed, 183 insertions(+), 56 deletions(-) create mode 100644 tools/dockerfile/grpc_sanity/Dockerfile rename tools/run_tests/{ => sanity}/check_cache_mk.sh (100%) rename tools/run_tests/{ => sanity}/check_sources_and_headers.py (66%) rename tools/run_tests/{ => sanity}/check_submodules.sh (98%) rename tools/run_tests/{ => sanity}/sanity_tests.yaml (61%) diff --git a/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile b/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile index 48541b00adf..ad11733b7de 100644 --- a/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile +++ b/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile @@ -95,8 +95,6 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ RUN cd /var/local && wget www.nuget.org/NuGet.exe ENV NUGET mono /var/local/NuGet.exe -# TODO(jtattermusch): add dependencies for other languages - ################## # Node dependencies diff --git a/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile b/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile index 2323f233875..067c209db76 100644 --- a/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile +++ b/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile @@ -95,8 +95,6 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ RUN cd /var/local && wget www.nuget.org/NuGet.exe ENV NUGET mono /var/local/NuGet.exe -# TODO(jtattermusch): add dependencies for other languages - ################## # Node dependencies diff --git a/tools/dockerfile/grpc_sanity/Dockerfile b/tools/dockerfile/grpc_sanity/Dockerfile new file mode 100644 index 00000000000..87b3a7a81f3 --- /dev/null +++ b/tools/dockerfile/grpc_sanity/Dockerfile @@ -0,0 +1,92 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# A work-in-progress Dockerfile that allows running gRPC test suites +# inside a docker container. + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +################## +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang + +################## +# Sanity test dependencies +RUN apt-get update && apt-get install -y python-pip +RUN pip install simplejson mako + +################## +# Docker "inception". +# Note this is quite the ugly hack. +# This makes sure that the docker binary we inject has its dependencies. +RUN curl https://get.docker.com/ | sh +RUN apt-get remove --purge -y docker-engine + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 148a0f5e08e..26e5585d2d8 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015, Google Inc. +# Copyright 2015-2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -43,8 +43,12 @@ chown `whoami` $XDG_CACHE_HOME mkdir -p /var/local/git git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc -nvm use 0.12 -rvm use ruby-2.1 +nvm use 0.12 || true + +if [ -x "$(command -v rvm)" ] +then + rvm use ruby-2.1 +fi mkdir -p reports diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 488af3b4da7..d3e17c51001 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -183,6 +183,9 @@ class CLanguage(object): def supports_multi_config(self): return True + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return self.make_target @@ -215,6 +218,9 @@ class NodeLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'node' @@ -246,6 +252,9 @@ class PhpLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'php' @@ -299,6 +308,9 @@ class PythonLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'python' @@ -330,6 +342,9 @@ class RubyLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'ruby' @@ -412,6 +427,9 @@ class CSharpLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'csharp' @@ -443,6 +461,9 @@ class ObjCLanguage(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return 'objc' @@ -451,8 +472,10 @@ class Sanity(object): def test_specs(self, config, args): import yaml - with open('tools/run_tests/sanity_tests.yaml', 'r') as f: - return [config.job_spec(cmd['script'].split(), None, timeout_seconds=None, environ={'TEST': 'true'}, cpu_cost=cmd.get('cpu_cost', 1)) + with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f: + return [config.job_spec(cmd['script'].split(), None, + timeout_seconds=None, environ={'TEST': 'true'}, + cpu_cost=cmd.get('cpu_cost', 1)) for cmd in yaml.load(f)] def pre_build_steps(self): @@ -476,6 +499,9 @@ class Sanity(object): def supports_multi_config(self): return False + def dockerfile_dir(self, config, arch): + return 'tools/dockerfile/grpc_sanity' + def __str__(self): return 'sanity' @@ -506,6 +532,9 @@ class Build(object): def supports_multi_config(self): return True + def dockerfile_dir(self, config, arch): + return None + def __str__(self): return self.make_target @@ -545,7 +574,7 @@ def _windows_arch_option(arch): else: print 'Architecture %s not supported.' % arch sys.exit(1) - + def _check_arch_option(arch): """Checks that architecture option is valid.""" @@ -595,15 +624,19 @@ def _windows_toolset_option(compiler): sys.exit(1) -def _get_dockerfile_dir(arch): +def _get_dockerfile_dir(language, cfg, arch): """Returns dockerfile to use""" - if arch == 'default' or arch == 'x64': - return 'tools/dockerfile/grpc_jenkins_slave_x64' - elif arch == 'x86': - return 'tools/dockerfile/grpc_jenkins_slave_x86' + custom = language.dockerfile_dir(cfg, arch) + if custom: + return custom else: - print 'Architecture %s not supported with current settings.' % arch - sys.exit(1) + if arch == 'default' or arch == 'x64': + return 'tools/dockerfile/grpc_jenkins_slave_x64' + elif arch == 'x86': + return 'tools/dockerfile/grpc_jenkins_slave_x86' + else: + print 'Architecture %s not supported with current settings.' % arch + sys.exit(1) def runs_per_test_type(arg_str): """Auxilary function to parse the "runs_per_test" flag. @@ -775,7 +808,9 @@ if args.use_docker: env = os.environ.copy() env['RUN_TESTS_COMMAND'] = run_tests_cmd - env['DOCKERFILE_DIR'] = _get_dockerfile_dir(args.arch) + env['DOCKERFILE_DIR'] = _get_dockerfile_dir(next(iter(languages)), + next(iter(build_configs)), + args.arch) env['DOCKER_RUN_SCRIPT'] = 'tools/jenkins/docker_run_tests.sh' if args.xml_report: env['XML_REPORT'] = args.xml_report @@ -786,7 +821,7 @@ if args.use_docker: shell=True, env=env) sys.exit(0) - + if platform_string() != 'windows' and args.compiler != 'default': print 'Compiler %s not supported on current platform.' % args.compiler sys.exit(1) diff --git a/tools/run_tests/check_cache_mk.sh b/tools/run_tests/sanity/check_cache_mk.sh similarity index 100% rename from tools/run_tests/check_cache_mk.sh rename to tools/run_tests/sanity/check_cache_mk.sh diff --git a/tools/run_tests/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py similarity index 66% rename from tools/run_tests/check_sources_and_headers.py rename to tools/run_tests/sanity/check_sources_and_headers.py index 50574f44b0e..3974af0032b 100755 --- a/tools/run_tests/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -33,9 +33,9 @@ import os import re import sys -root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +root = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) with open(os.path.join(root, 'tools', 'run_tests', 'sources_and_headers.json')) as f: - js = json.loads(f.read()) + js = json.loads(f.read()) re_inc1 = re.compile(r'^#\s*include\s*"([^"]*)"') assert re_inc1.match('#include "foo"').group(1) == 'foo' @@ -43,41 +43,41 @@ re_inc2 = re.compile(r'^#\s*include\s*<((grpc|grpc\+\+)/[^"]*)>') assert re_inc2.match('#include ').group(1) == 'grpc++/foo' def get_target(name): - for target in js: - if target['name'] == name: - return target - assert False, 'no target %s' % name + for target in js: + if target['name'] == name: + return target + assert False, 'no target %s' % name def target_has_header(target, name): -# print target['name'], name - if name in target['headers']: - return True - for dep in target['deps']: - if target_has_header(get_target(dep), name): - return True - if name == 'src/core/profiling/stap_probes.h': - return True - return False + # print target['name'], name + if name in target['headers']: + return True + for dep in target['deps']: + if target_has_header(get_target(dep), name): + return True + if name == 'src/core/profiling/stap_probes.h': + return True + return False errors = 0 for target in js: - for fn in target['src']: - with open(os.path.join(root, fn)) as f: - src = f.read().splitlines() - for line in src: - m = re_inc1.match(line) - if m: - if not target_has_header(target, m.group(1)): - print ( - 'target %s (%s) does not name header %s as a dependency' % ( - target['name'], fn, m.group(1))) - errors += 1 - m = re_inc2.match(line) - if m: - if not target_has_header(target, 'include/' + m.group(1)): - print ( - 'target %s (%s) does not name header %s as a dependency' % ( - target['name'], fn, m.group(1))) - errors += 1 + for fn in target['src']: + with open(os.path.join(root, fn)) as f: + src = f.read().splitlines() + for line in src: + m = re_inc1.match(line) + if m: + if not target_has_header(target, m.group(1)): + print ( + 'target %s (%s) does not name header %s as a dependency' % ( + target['name'], fn, m.group(1))) + errors += 1 + m = re_inc2.match(line) + if m: + if not target_has_header(target, 'include/' + m.group(1)): + print ( + 'target %s (%s) does not name header %s as a dependency' % ( + target['name'], fn, m.group(1))) + errors += 1 assert errors == 0 diff --git a/tools/run_tests/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh similarity index 98% rename from tools/run_tests/check_submodules.sh rename to tools/run_tests/sanity/check_submodules.sh index b4ca4fa3ce5..f49230e49aa 100755 --- a/tools/run_tests/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -34,7 +34,7 @@ set -e export TEST=true -cd `dirname $0`/../.. +cd `dirname $0`/../../.. submodules=`mktemp /tmp/submXXXXXX` want_submodules=`mktemp /tmp/submXXXXXX` diff --git a/tools/run_tests/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml similarity index 61% rename from tools/run_tests/sanity_tests.yaml rename to tools/run_tests/sanity/sanity_tests.yaml index 757bcd9d341..809e6ce6454 100644 --- a/tools/run_tests/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -1,7 +1,7 @@ # a set of tests that are run in parallel for sanity tests -- script: tools/run_tests/check_sources_and_headers.py -- script: tools/run_tests/check_submodules.sh -- script: tools/run_tests/check_cache_mk.sh +- script: tools/run_tests/sanity/check_sources_and_headers.py +- script: tools/run_tests/sanity/check_submodules.sh +- script: tools/run_tests/sanity/check_cache_mk.sh - script: tools/buildgen/generate_projects.sh -j 3 cpu_cost: 3 - script: tools/distrib/check_copyright.py From 2bb4d3fda259d292b5373bbc71dbe4ffac85895e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 26 Jan 2016 14:29:51 -0800 Subject: [PATCH 06/11] dockerfile cleanup --- .../dockerfile/grpc_jenkins_slave_x64/Dockerfile | 13 +------------ .../dockerfile/grpc_jenkins_slave_x86/Dockerfile | 6 +----- tools/dockerfile/grpc_sanity/Dockerfile | 15 +-------------- 3 files changed, 3 insertions(+), 31 deletions(-) diff --git a/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile b/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile index ad11733b7de..7785f3d99ae 100644 --- a/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile +++ b/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile @@ -27,8 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# A work-in-progress Dockerfile that allows running gRPC test suites -# inside a docker container. +# Dockerfile for running gRPC test suites inside a docker container. FROM debian:jessie @@ -152,18 +151,8 @@ RUN apt-get update && apt-get install -y \ ################## # Zookeeper dependencies - -# Install dependencies - RUN apt-get install -y libzookeeper-mt-dev -################## -# Docker "inception". -# Note this is quite the ugly hack. -# This makes sure that the docker binary we inject has its dependencies. -RUN curl https://get.docker.com/ | sh -RUN apt-get remove --purge -y docker-engine - RUN mkdir /var/local/jenkins # Define the default command. diff --git a/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile b/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile index 067c209db76..c5cd45ea088 100644 --- a/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile +++ b/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile @@ -27,8 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# A work-in-progress Dockerfile that allows running gRPC test suites -# inside a docker container. +# Dockerfile for running gRPC test suites inside a docker container. FROM 32bit/debian:jessie @@ -151,9 +150,6 @@ RUN apt-get update && apt-get install -y \ ################## # Zookeeper dependencies - -# Install dependencies - RUN apt-get install -y libzookeeper-mt-dev diff --git a/tools/dockerfile/grpc_sanity/Dockerfile b/tools/dockerfile/grpc_sanity/Dockerfile index 87b3a7a81f3..6f7797d32ab 100644 --- a/tools/dockerfile/grpc_sanity/Dockerfile +++ b/tools/dockerfile/grpc_sanity/Dockerfile @@ -27,8 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# A work-in-progress Dockerfile that allows running gRPC test suites -# inside a docker container. +# Dockerfile for running gRPC sanity tests FROM debian:jessie @@ -62,18 +61,6 @@ RUN apt-get update && apt-get install -y \ wget \ zip && apt-get clean -# Prepare ccache -RUN ln -s /usr/bin/ccache /usr/local/bin/gcc -RUN ln -s /usr/bin/ccache /usr/local/bin/g++ -RUN ln -s /usr/bin/ccache /usr/local/bin/cc -RUN ln -s /usr/bin/ccache /usr/local/bin/c++ -RUN ln -s /usr/bin/ccache /usr/local/bin/clang -RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ - -################## -# C++ dependencies -RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang - ################## # Sanity test dependencies RUN apt-get update && apt-get install -y python-pip From ed342b1c4d9abe15dd9f8f29f38bb3b46fd143c8 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 26 Jan 2016 14:40:31 -0800 Subject: [PATCH 07/11] meaningful name for grpc_jenkins_slave --- .../Dockerfile | 0 .../Dockerfile | 0 tools/run_tests/run_tests.py | 6 +++--- 3 files changed, 3 insertions(+), 3 deletions(-) rename tools/dockerfile/{grpc_jenkins_slave_x64 => grpc_tests_multilang_x64}/Dockerfile (100%) rename tools/dockerfile/{grpc_jenkins_slave_x86 => grpc_tests_multilang_x86}/Dockerfile (100%) diff --git a/tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile b/tools/dockerfile/grpc_tests_multilang_x64/Dockerfile similarity index 100% rename from tools/dockerfile/grpc_jenkins_slave_x64/Dockerfile rename to tools/dockerfile/grpc_tests_multilang_x64/Dockerfile diff --git a/tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile b/tools/dockerfile/grpc_tests_multilang_x86/Dockerfile similarity index 100% rename from tools/dockerfile/grpc_jenkins_slave_x86/Dockerfile rename to tools/dockerfile/grpc_tests_multilang_x86/Dockerfile diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index d3e17c51001..885a79c07b0 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -631,9 +631,9 @@ def _get_dockerfile_dir(language, cfg, arch): return custom else: if arch == 'default' or arch == 'x64': - return 'tools/dockerfile/grpc_jenkins_slave_x64' + return 'tools/dockerfile/grpc_tests_multilang_x64' elif arch == 'x86': - return 'tools/dockerfile/grpc_jenkins_slave_x86' + return 'tools/dockerfile/grpc_tests_multilang_x86' else: print 'Architecture %s not supported with current settings.' % arch sys.exit(1) @@ -804,7 +804,7 @@ if args.use_docker: time.sleep(5) child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] - run_tests_cmd = 'tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) + run_tests_cmd = 'python tools/run_tests/run_tests.py %s' % ' '.join(child_argv[1:]) env = os.environ.copy() env['RUN_TESTS_COMMAND'] = run_tests_cmd From 154b0eeaa909143f9c227c76b2afe5fe6b285c02 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 25 Jan 2016 16:45:29 -0800 Subject: [PATCH 08/11] Enable binary linux distributions for Python --- setup.py | 5 +- src/python/grpcio/commands.py | 130 ++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 52d3d2241ef..01e91808543 100644 --- a/setup.py +++ b/setup.py @@ -119,6 +119,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( + 'six>=1.10', 'enum34>=1.0.4', 'futures>=2.2.0', # TODO(atash): eventually split the grpcio package into a metapackage @@ -131,6 +132,7 @@ SETUP_REQUIRES = ( ) + INSTALL_REQUIRES COMMAND_CLASS = { + 'install': commands.Install, 'doc': commands.SphinxDocumentation, 'build_proto_modules': commands.BuildProtoModules, 'build_project_metadata': commands.BuildProjectMetadata, @@ -138,6 +140,7 @@ COMMAND_CLASS = { 'build_ext': commands.BuildExt, 'gather': commands.Gather, 'run_interop': commands.RunInterop, + 'bdist_egg_grpc_custom': commands.BdistEggCustomName, } # Ensure that package data is copied over before any commands have been run: @@ -187,7 +190,7 @@ else: setuptools.setup( name='grpcio', - version='0.12.0b5', + version='0.12.0b6', license=LICENSE, ext_modules=CYTHON_EXTENSION_MODULES, packages=list(PACKAGES), diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 29b506abe0a..a6b8ad3fc05 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -30,15 +30,22 @@ """Provides distutils command classes for the GRPC Python setup process.""" import distutils +import glob import os import os.path +import platform import re +import shutil import subprocess import sys +import traceback import setuptools +from setuptools.command import bdist_egg from setuptools.command import build_ext from setuptools.command import build_py +from setuptools.command import easy_install +from setuptools.command import install from setuptools.command import test import support @@ -58,6 +65,129 @@ class CommandError(Exception): """Simple exception class for GRPC custom commands.""" +# TODO(atash): Remove this once PyPI has better Linux bdist support. See +# https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported +def _get_linux_bdist_egg(decorated_basename, target_egg_basename): + """Returns a string path to a .egg file for Linux to install. + + If we can retrieve a pre-compiled egg from online, uses it. Else, emits a + warning and builds from source. + """ + # Break import style to ensure that setup.py has had a chance to install the + # relevant package eggs. + from six.moves.urllib import request + decorated_path = decorated_basename + '.egg' + try: + url = ( + 'https://storage.googleapis.com/grpc-precompiled-binaries/' + 'python/{target}' + .format(target=decorated_path)) + egg_data = request.urlopen(url).read() + except IOError as error: + raise CommandError( + '{}\n\nCould not find the bdist egg {}: {}' + .format(traceback.format_exc(), decorated_path, error.message)) + # Our chosen local egg path. + egg_path = target_egg_basename + '.egg' + try: + with open(egg_path, 'w') as egg_file: + egg_file.write(egg_data) + except IOError as error: + raise CommandError( + '{}\n\nCould not write grpcio egg: {}' + .format(traceback.format_exc(), error.message)) + return egg_path + + +class EggNameMixin(object): + + def egg_name(self, with_custom): + """ + Args: + with_custom: Boolean describing whether or not to decorate the egg name + with custom gRPC-specific target information. + """ + egg_command = self.get_finalized_command('bdist_egg') + base = os.path.splitext(os.path.basename(egg_command.egg_output))[0] + if with_custom: + flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4' + return '{base}-{flavor}'.format(base=base, flavor=flavor) + else: + return base + + +class Install(install.install, EggNameMixin): + """Custom Install command for gRPC Python. + + This is for bdist shims and whatever else we might need a custom install + command for. + """ + + user_options = install.install.user_options + [ + # TODO(atash): remove this once manylinux gets on PyPI. See + # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported + ('use-linux-bdist', None, + 'Whether to retrieve a binary for Linux instead of building from ' + 'source.'), + ] + + def initialize_options(self): + install.install.initialize_options(self) + self.use_linux_bdist = False + + def finalize_options(self): + install.install.finalize_options(self) + + def run(self): + if self.use_linux_bdist: + try: + egg_path = _get_linux_bdist_egg(self.egg_name(True), + self.egg_name(False)) + except CommandError as error: + sys.stderr.write( + '\nWARNING: Failed to acquire grpcio prebuilt binary:\n' + '{}.\n\n'.format(error.message)) + raise + try: + self._run_bdist_retrieval_install(egg_path) + except Exception as error: + # if anything else happens (and given how there's no way to really know + # what's happening in setuptools here, I mean *anything*), warn the user + # and fall back to building from source. + sys.stderr.write( + '{}\nWARNING: Failed to install grpcio prebuilt binary.\n\n' + .format(traceback.format_exc())) + install.install.run(self) + else: + install.install.run(self) + + # TODO(atash): Remove this once PyPI has better Linux bdist support. See + # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported + def _run_bdist_retrieval_install(self, bdist_egg): + easy_install = self.distribution.get_command_class('easy_install') + easy_install_command = easy_install( + self.distribution, args='x', root=self.root, record=self.record, + ) + easy_install_command.ensure_finalized() + easy_install_command.always_copy_from = '.' + easy_install_command.package_index.scan(glob.glob('*.egg')) + arguments = [bdist_egg] + if setuptools.bootstrap_install_from: + args.insert(0, setuptools.bootstrap_install_from) + easy_install_command.args = arguments + easy_install_command.run() + setuptools.bootstrap_install_from = None + + +class BdistEggCustomName(bdist_egg.bdist_egg, EggNameMixin): + """Thin wrapper around the bdist_egg command to build with our custom name.""" + + def run(self): + bdist_egg.bdist_egg.run(self) + target = os.path.join(self.dist_dir, '{}.egg'.format(self.egg_name(True))) + shutil.move(self.get_outputs()[0], target) + + class SphinxDocumentation(setuptools.Command): """Command to generate documentation via sphinx.""" From 16e45702ac13ce756a911e492317856e94d067aa Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 27 Jan 2016 14:51:12 -0800 Subject: [PATCH 09/11] clang-format --- test/cpp/qps/client_async.cc | 34 ++++++++++++++-------------------- test/cpp/qps/server_async.cc | 20 ++++++++------------ 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/test/cpp/qps/client_async.cc b/test/cpp/qps/client_async.cc index 4ab74a9d4c6..e423ee2598a 100644 --- a/test/cpp/qps/client_async.cc +++ b/test/cpp/qps/client_async.cc @@ -94,8 +94,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function< std::unique_ptr>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*)> - start_req, + CompletionQueue*)> start_req, std::function on_done) : ClientRpcContext(channel_id), context_(), @@ -141,8 +140,7 @@ class ClientRpcContextUnaryImpl : public ClientRpcContext { std::function callback_; std::function>( BenchmarkService::Stub*, grpc::ClientContext*, const RequestType&, - CompletionQueue*)> - start_req_; + CompletionQueue*)> start_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -161,12 +159,11 @@ class AsyncClient : public ClientImpl { using Client::closed_loop_; using ClientImpl::channels_; using ClientImpl::request_; - AsyncClient( - const ClientConfig& config, - std::function - setup_ctx, - std::function(std::shared_ptr)> - create_stub) + AsyncClient(const ClientConfig& config, + std::function setup_ctx, + std::function(std::shared_ptr)> + create_stub) : ClientImpl(config, create_stub), num_async_threads_(NumThreads(config)), channel_lock_(new std::mutex[config.client_channels()]), @@ -408,8 +405,7 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { std::function>( BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, - void*)> - start_req, + void*)> start_req, std::function on_done) : ClientRpcContext(channel_id), context_(), @@ -461,10 +457,10 @@ class ClientRpcContextStreamingImpl : public ClientRpcContext { ResponseType response_; bool (ClientRpcContextStreamingImpl::*next_state_)(bool, Histogram*); std::function callback_; - std::function>( - BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, void*)> - start_req_; + std::function< + std::unique_ptr>( + BenchmarkService::Stub*, grpc::ClientContext*, CompletionQueue*, + void*)> start_req_; grpc::Status status_; double start_; std::unique_ptr> @@ -508,8 +504,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { int channel_id, grpc::GenericStub* stub, const ByteBuffer& req, std::function( grpc::GenericStub*, grpc::ClientContext*, - const grpc::string& method_name, CompletionQueue*, void*)> - start_req, + const grpc::string& method_name, CompletionQueue*, void*)> start_req, std::function on_done) : ClientRpcContext(channel_id), context_(), @@ -566,8 +561,7 @@ class ClientRpcContextGenericStreamingImpl : public ClientRpcContext { std::function callback_; std::function( grpc::GenericStub*, grpc::ClientContext*, const grpc::string&, - CompletionQueue*, void*)> - start_req_; + CompletionQueue*, void*)> start_req_; grpc::Status status_; double start_; std::unique_ptr stream_; diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc index ddfeb8638d5..ffa62263881 100644 --- a/test/cpp/qps/server_async.cc +++ b/test/cpp/qps/server_async.cc @@ -72,8 +72,7 @@ class AsyncQpsServerTest : public Server { CompletionQueue *, ServerCompletionQueue *, void *)> request_streaming_function, std::function - process_rpc) + ResponseType *)> process_rpc) : Server(config) { char *server_address = NULL; @@ -191,8 +190,7 @@ class AsyncQpsServerTest : public Server { ServerRpcContextUnaryImpl( std::function *, - void *)> - request_method, + void *)> request_method, std::function invoke_method) : srv_ctx_(new ServerContextType), @@ -389,14 +387,12 @@ static Status ProcessGenericRPC(const PayloadConfig &payload_config, } std::unique_ptr CreateAsyncServer(const ServerConfig &config) { - return std::unique_ptr( - new AsyncQpsServerTest( - config, RegisterBenchmarkService, - &BenchmarkService::AsyncService::RequestUnaryCall, - &BenchmarkService::AsyncService::RequestStreamingCall, - ProcessSimpleRPC)); + return std::unique_ptr(new AsyncQpsServerTest< + SimpleRequest, SimpleResponse, BenchmarkService::AsyncService, + grpc::ServerContext>( + config, RegisterBenchmarkService, + &BenchmarkService::AsyncService::RequestUnaryCall, + &BenchmarkService::AsyncService::RequestStreamingCall, ProcessSimpleRPC)); } std::unique_ptr CreateAsyncGenericServer(const ServerConfig &config) { return std::unique_ptr( From b9c5505c4a88c633044e645c3bd92eeb362b11c7 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 27 Jan 2016 15:22:48 -0800 Subject: [PATCH 10/11] Fix undefined behavior --- src/core/client_config/subchannel.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/core/client_config/subchannel.c b/src/core/client_config/subchannel.c index 2992da8b79d..67fada080f4 100644 --- a/src/core/client_config/subchannel.c +++ b/src/core/client_config/subchannel.c @@ -284,9 +284,13 @@ grpc_subchannel *grpc_subchannel_create(grpc_connector *connector, c->connector = connector; grpc_connector_ref(c->connector); c->num_filters = args->filter_count; - c->filters = gpr_malloc(sizeof(grpc_channel_filter *) * c->num_filters); - memcpy((void *)c->filters, args->filters, - sizeof(grpc_channel_filter *) * c->num_filters); + if (c->num_filters > 0) { + c->filters = gpr_malloc(sizeof(grpc_channel_filter *) * c->num_filters); + memcpy((void *)c->filters, args->filters, + sizeof(grpc_channel_filter *) * c->num_filters); + } else { + c->filters = NULL; + } c->addr = gpr_malloc(args->addr_len); memcpy(c->addr, args->addr, args->addr_len); grpc_pollset_set_init(&c->pollset_set); @@ -483,7 +487,9 @@ static void publish_transport(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) { /* build final filter list */ num_filters = c->num_filters + c->connecting_result.num_filters + 1; filters = gpr_malloc(sizeof(*filters) * num_filters); - memcpy((void *)filters, c->filters, sizeof(*filters) * c->num_filters); + if (c->num_filters > 0) { + memcpy((void *)filters, c->filters, sizeof(*filters) * c->num_filters); + } memcpy((void *)(filters + c->num_filters), c->connecting_result.filters, sizeof(*filters) * c->connecting_result.num_filters); filters[num_filters - 1] = &grpc_connected_channel_filter; From 3e301a37bcc1a1d4e34caf2fbb446e4e26e1f8a8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 27 Jan 2016 15:36:52 -0800 Subject: [PATCH 11/11] Reduce CPU, increase MSAN timeouts --- Makefile | 2 +- build.yaml | 2 +- tools/jenkins/run_jenkins.sh | 11 +++++++---- tools/run_tests/configs.json | 2 +- tools/run_tests/jobset.py | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 8dfb505424d..0ba7067973f 100644 --- a/Makefile +++ b/Makefile @@ -208,7 +208,7 @@ CFLAGS_msan = -O0 -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-fr CXXFLAGS_msan = -O0 -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) DEFINES_msan = NDEBUG -DEFINES_msan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=1.5 +DEFINES_msan += GRPC_TEST_SLOWDOWN_BUILD_FACTOR=2 VALID_CONFIG_mutrace = 1 CC_mutrace = $(DEFAULT_CC) diff --git a/build.yaml b/build.yaml index 36b031003a8..8655027363c 100644 --- a/build.yaml +++ b/build.yaml @@ -2561,7 +2561,7 @@ configs: -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) LDXX: clang++ compile_the_world: true - timeout_multiplier: 1.5 + timeout_multiplier: 2 mutrace: CPPFLAGS: -O0 DEFINES: _DEBUG DEBUG diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index 49b2fa55c2d..c9bc4f18945 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -42,6 +42,9 @@ set -ex -o igncr || set -ex # Grabbing the machine's architecture arch=`uname -m` +# Number of concurrent cores each executor should try and schedule +EXECUTOR_CORES=2 + case $platform in i386) arch="i386" @@ -54,7 +57,7 @@ if [ "$platform" == "linux" ] then echo "building $language on Linux" - ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" + ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml -j $EXECUTOR_CORES $@ || TESTS_FAILED="true" elif [ "$platform" == "windows" ] then @@ -63,7 +66,7 @@ then # Prevent msbuild from picking up "platform" env variable, which would break the build unset platform - python tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" + python tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j $EXECUTOR_CORES $@ || TESTS_FAILED="true" elif [ "$platform" == "macos" ] then @@ -72,13 +75,13 @@ then # Prevent msbuild from picking up "platform" env variable, which would break the build unset platform - ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" + ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j $EXECUTOR_CORES $@ || TESTS_FAILED="true" elif [ "$platform" == "freebsd" ] then echo "building $language on FreeBSD" - MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j 3 $@ || TESTS_FAILED="true" + MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml -j $EXECUTOR_CORES $@ || TESTS_FAILED="true" else echo "Unknown platform $platform" diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json index 769942df996..d508c6394cb 100644 --- a/tools/run_tests/configs.json +++ b/tools/run_tests/configs.json @@ -59,7 +59,7 @@ }, { "config": "msan", - "timeout_multiplier": 1.5 + "timeout_multiplier": 2 }, { "config": "mutrace" diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index beeb99c0ae9..adf178bb3c3 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -360,7 +360,7 @@ class Jobset(object): if self.cancelled(): return False current_cpu_cost = self.cpu_cost() if current_cpu_cost == 0: break - if current_cpu_cost + spec.cpu_cost < self._maxjobs: break + if current_cpu_cost + spec.cpu_cost <= self._maxjobs: break self.reap() if self.cancelled(): return False if spec.hash_targets: