diff --git a/src/python/grpcio/README.rst b/src/python/grpcio/README.rst index cb3f6b87fe4..afc4fe6a370 100644 --- a/src/python/grpcio/README.rst +++ b/src/python/grpcio/README.rst @@ -48,6 +48,7 @@ package named :code:`python-dev`). $ export REPO_ROOT=grpc # REPO_ROOT can be any directory of your choice $ git clone https://github.com/grpc/grpc.git $REPO_ROOT $ cd $REPO_ROOT + $ git submodule update --init # For the next two commands do `sudo pip install` if you get permission-denied errors $ pip install -rrequirements.txt diff --git a/src/python/grpcio/tests/stress/client.py b/src/python/grpcio/tests/stress/client.py index a733741b736..e2e016760c8 100644 --- a/src/python/grpcio/tests/stress/client.py +++ b/src/python/grpcio/tests/stress/client.py @@ -117,7 +117,10 @@ def run_test(args): for runner in runners: runner.start() try: - raise exception_queue.get(block=True, timeout=args.test_duration_secs) + timeout_secs = args.test_duration_secs + if timeout_secs < 0: + timeout_secs = None + raise exception_queue.get(block=True, timeout=timeout_secs) except Queue.Empty: # No exceptions thrown, success pass diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 07f7bb93b8a..6d65db83067 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -78,9 +78,11 @@ output_dir = File.expand_path(RbConfig::CONFIG['topdir']) grpc_lib_dir = File.join(output_dir, 'libs', grpc_config) ENV['BUILDDIR'] = output_dir -puts 'Building internal gRPC into ' + grpc_lib_dir -system("make -j -C #{grpc_root} #{grpc_lib_dir}/libgrpc.a CONFIG=#{grpc_config}") -exit 1 unless $? == 0 +unless windows + puts 'Building internal gRPC into ' + grpc_lib_dir + system("make -j -C #{grpc_root} #{grpc_lib_dir}/libgrpc.a CONFIG=#{grpc_config}") + exit 1 unless $? == 0 +end $CFLAGS << ' -I' + File.join(grpc_root, 'include') $LDFLAGS << ' ' + File.join(grpc_lib_dir, 'libgrpc.a') unless windows diff --git a/src/ruby/ext/grpc/rb_byte_buffer.c b/src/ruby/ext/grpc/rb_byte_buffer.c index cba910d832a..1172691116c 100644 --- a/src/ruby/ext/grpc/rb_byte_buffer.c +++ b/src/ruby/ext/grpc/rb_byte_buffer.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_byte_buffer.h" -#include - #include #include #include diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c index 48c49a21e91..1b06273af9d 100644 --- a/src/ruby/ext/grpc/rb_call.c +++ b/src/ruby/ext/grpc/rb_call.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_call.h" -#include - #include #include diff --git a/src/ruby/ext/grpc/rb_call_credentials.c b/src/ruby/ext/grpc/rb_call_credentials.c index 38bf1f7710f..79ca5b32ced 100644 --- a/src/ruby/ext/grpc/rb_call_credentials.c +++ b/src/ruby/ext/grpc/rb_call_credentials.c @@ -32,10 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_call_credentials.h" -#include #include #include @@ -86,11 +86,11 @@ static VALUE grpc_rb_call_credentials_callback_rescue(VALUE args, rb_funcall(exception_object, rb_intern("backtrace"), 0), rb_intern("join"), 1, rb_str_new2("\n\tfrom ")); - VALUE exception_info = rb_funcall(exception_object, rb_intern("to_s"), 0); + VALUE rb_exception_info = rb_funcall(exception_object, rb_intern("to_s"), 0); const char *exception_classname = rb_obj_classname(exception_object); (void)args; gpr_log(GPR_INFO, "Call credentials callback failed: %s: %s\n%s", - exception_classname, StringValueCStr(exception_info), + exception_classname, StringValueCStr(rb_exception_info), StringValueCStr(backtrace)); rb_hash_aset(result, rb_str_new2("metadata"), Qnil); /* Currently only gives the exception class name. It should be possible get diff --git a/src/ruby/ext/grpc/rb_channel.c b/src/ruby/ext/grpc/rb_channel.c index 984afad1073..013321ffc8a 100644 --- a/src/ruby/ext/grpc/rb_channel.c +++ b/src/ruby/ext/grpc/rb_channel.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_channel.h" -#include - #include #include #include diff --git a/src/ruby/ext/grpc/rb_channel_args.c b/src/ruby/ext/grpc/rb_channel_args.c index 2ffb8f41dae..87c0e0a7055 100644 --- a/src/ruby/ext/grpc/rb_channel_args.c +++ b/src/ruby/ext/grpc/rb_channel_args.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_channel_args.h" -#include - #include #include "rb_grpc.h" diff --git a/src/ruby/ext/grpc/rb_channel_credentials.c b/src/ruby/ext/grpc/rb_channel_credentials.c index 09bd3093a94..cbb23885aa6 100644 --- a/src/ruby/ext/grpc/rb_channel_credentials.c +++ b/src/ruby/ext/grpc/rb_channel_credentials.c @@ -31,14 +31,13 @@ * */ +#include + #include -#include #include "rb_grpc_imports.generated.h" #include "rb_channel_credentials.h" -#include - #include #include #include diff --git a/src/ruby/ext/grpc/rb_completion_queue.c b/src/ruby/ext/grpc/rb_completion_queue.c index 2a2eee190cd..4bb615f8bec 100644 --- a/src/ruby/ext/grpc/rb_completion_queue.c +++ b/src/ruby/ext/grpc/rb_completion_queue.c @@ -32,10 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_completion_queue.h" -#include #include #include diff --git a/src/ruby/ext/grpc/rb_event_thread.c b/src/ruby/ext/grpc/rb_event_thread.c index 2649a1087f2..9e85bbcfbf2 100644 --- a/src/ruby/ext/grpc/rb_event_thread.c +++ b/src/ruby/ext/grpc/rb_event_thread.c @@ -32,12 +32,12 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_event_thread.h" #include -#include #include #include #include diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c index acb47b00558..06a07ac6463 100644 --- a/src/ruby/ext/grpc/rb_grpc.c +++ b/src/ruby/ext/grpc/rb_grpc.c @@ -32,11 +32,11 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_grpc.h" #include -#include #include #include diff --git a/src/ruby/ext/grpc/rb_server.c b/src/ruby/ext/grpc/rb_server.c index 96e60c67763..2b3acaaf59c 100644 --- a/src/ruby/ext/grpc/rb_server.c +++ b/src/ruby/ext/grpc/rb_server.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_server.h" -#include - #include #include #include "rb_call.h" diff --git a/src/ruby/ext/grpc/rb_server_credentials.c b/src/ruby/ext/grpc/rb_server_credentials.c index b2d7280a30a..3b0fb6c910d 100644 --- a/src/ruby/ext/grpc/rb_server_credentials.c +++ b/src/ruby/ext/grpc/rb_server_credentials.c @@ -32,11 +32,10 @@ */ #include + #include "rb_grpc_imports.generated.h" #include "rb_server_credentials.h" -#include - #include #include diff --git a/templates/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile.template b/templates/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile.template new file mode 100644 index 00000000000..27e9eeec5a8 --- /dev/null +++ b/templates/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile.template @@ -0,0 +1,45 @@ +%YAML 1.2 +--- | + # Copyright 2016, Google Inc. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are + # met: + # + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following disclaimer + # in the documentation and/or other materials provided with the + # distribution. + # * Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + FROM debian:jessie + + <%include file="../../apt_get_basic.include"/> + <%include file="../../ccache_setup.include"/> + <%include file="../../cxx_deps.include"/> + <%include file="../../gcp_api_libraries.include"/> + <%include file="../../python_deps.include"/> + + RUN pip install coverage + RUN pip install oauth2client + + # Define the default command. + CMD ["bash"] + diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile new file mode 100644 index 00000000000..606b7654576 --- /dev/null +++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/Dockerfile @@ -0,0 +1,103 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + +#================= +# C++ dependencies +RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean + +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client + + +#==================== +# Python dependencies + +# Install dependencies + +RUN apt-get update && apt-get install -y \ + python-all-dev \ + python3-all-dev \ + python-pip + +# Install Python packages from PyPI +RUN pip install pip --upgrade +RUN pip install virtualenv +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 tox + + +RUN pip install coverage +RUN pip install oauth2client + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh new file mode 100755 index 00000000000..e65332f2f30 --- /dev/null +++ b/tools/dockerfile/stress_test/grpc_interop_stress_python/build_interop_stress.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds Python interop server and client in a base image. +set -e + +mkdir -p /var/local/git +git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc + +# copy service account keys if available +cp -r /var/local/jenkins/service_account $HOME || true + +cd /var/local/git/grpc + +tools/run_tests/run_tests.py -l python -c opt --build_only + +# Build c++ interop client +make metrics_client -j + diff --git a/tools/run_tests/stress_test/configs/csharp.json b/tools/run_tests/stress_test/configs/csharp.json index 406949557dc..587e1552b9a 100644 --- a/tools/run_tests/stress_test/configs/csharp.json +++ b/tools/run_tests/stress_test/configs/csharp.json @@ -80,7 +80,7 @@ "buildDockerImages": true, "pollIntervalSecs": 60, "testDurationSecs": 7200, - "kubernetesProxyPort": 8001, + "kubernetesProxyPort": 8009, "datasetIdNamePrefix": "stress_test_csharp", "summaryTableId": "summary", "qpsTableId": "qps", diff --git a/tools/run_tests/stress_test/configs/python.json b/tools/run_tests/stress_test/configs/python.json new file mode 100644 index 00000000000..4f85de1d5f6 --- /dev/null +++ b/tools/run_tests/stress_test/configs/python.json @@ -0,0 +1,98 @@ +{ + "dockerImages": { + "grpc_stress_python" : { + "buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", + "dockerFileDir": "grpc_interop_stress_python" + } + }, + + "clientTemplates": { + "baseTemplates": { + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", + "pollIntervalSecs": 60, + "clientArgs": { + "num_channels_per_server":5, + "num_stubs_per_channel":10, + "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", + "metrics_port": 8081 + }, + "metricsPort": 8081, + "metricsArgs": { + "metrics_server_address": "localhost:8081", + "total_only": "true" + }, + "env": { + "PYTHONPATH": "/var/local/git/grpc/src/python/gens:/var/local/git/grpc/src/python/grpcio", + "LD_LIBRARY_PATH":"/var/local/git/grpc/libs/opt" + } + } + }, + "templates": { + "python_client": { + "baseTemplate": "default", + "stressClientCmd": [ + "python", + "/var/local/git/grpc/src/python/grpcio/tests/stress/client.py" + ], + "metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] + } + } + }, + + "serverTemplates": { + "baseTemplates":{ + "default": { + "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", + "serverPort": 8080, + "serverArgs": { + "port": 8080 + }, + "env": { + "PYTHONPATH": "/var/local/git/grpc/src/python/gens:/var/local/git/grpc/src/python/grpcio", + "LD_LIBRARY_PATH":"/var/local/git/grpc/libs/opt" + } + } + }, + "templates": { + "python_server": { + "baseTemplate": "default", + "stressServerCmd": [ + "python", + "/var/local/git/grpc/src/python/grpcio/tests/interop/server.py" + ] + } + } + }, + + "testMatrix": { + "serverPodSpecs": { + "python-stress-server": { + "serverTemplate": "python_server", + "dockerImage": "grpc_stress_python", + "numInstances": 1 + } + }, + + "clientPodSpecs": { + "python-stress-client": { + "clientTemplate": "python_client", + "dockerImage": "grpc_stress_python", + "numInstances": 5, + "serverPodSpec": "python-stress-server" + } + } + }, + + "globalSettings": { + "buildDockerImages": true, + "pollIntervalSecs": 60, + "testDurationSecs": 7200, + "kubernetesProxyPort": 8011, + "datasetIdNamePrefix": "stress_test_python", + "summaryTableId": "summary", + "qpsTableId": "qps", + "podWarmupSecs": 60 + } +} + diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index d4f1c4ad3dc..583e58316f4 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -69,7 +69,7 @@ class ClientTemplate: def __init__(self, name, stress_client_cmd, metrics_client_cmd, metrics_port, wrapper_script_path, poll_interval_secs, client_args_dict, - metrics_args_dict, will_run_forever): + metrics_args_dict, will_run_forever, env_dict): self.name = name self.stress_client_cmd = stress_client_cmd self.metrics_client_cmd = metrics_client_cmd @@ -79,19 +79,21 @@ class ClientTemplate: self.client_args_dict = client_args_dict self.metrics_args_dict = metrics_args_dict self.will_run_forever = will_run_forever + self.env_dict = env_dict class ServerTemplate: """ Contains all the common settings used by a stress server """ def __init__(self, name, server_cmd, wrapper_script_path, server_port, - server_args_dict, will_run_forever): + server_args_dict, will_run_forever, env_dict): self.name = name self.server_cmd = server_cmd self.wrapper_script_path = wrapper_script_path self.server_port = server_port self.server_args_dict = server_args_dict self.will_run_forever = will_run_forever + self.env_dict = env_dict class DockerImage: @@ -240,6 +242,7 @@ class Gke: # server_pod_spec.template.wrapper_script_path) are are injected into the # container via environment variables server_env = self.gke_env.copy() + server_env.update(server_pod_spec.template.env_dict) server_env.update({ 'STRESS_TEST_IMAGE_TYPE': 'SERVER', 'STRESS_TEST_CMD': server_pod_spec.template.server_cmd, @@ -283,6 +286,7 @@ class Gke: # client_pod_spec.template.wrapper_script_path) are are injected into the # container via environment variables client_env = self.gke_env.copy() + client_env.update(client_pod_spec.template.env_dict) client_env.update({ 'STRESS_TEST_IMAGE_TYPE': 'CLIENT', 'STRESS_TEST_CMD': client_pod_spec.template.stress_client_cmd, @@ -425,7 +429,8 @@ class Config: template_name, stress_client_cmd, metrics_client_cmd, temp_dict['metricsPort'], temp_dict['wrapperScriptPath'], temp_dict['pollIntervalSecs'], temp_dict['clientArgs'].copy(), - temp_dict['metricsArgs'].copy(), temp_dict.get('willRunForever', 1)) + temp_dict['metricsArgs'].copy(), temp_dict.get('willRunForever', 1), + temp_dict.get('env', {}).copy()) return client_templates_dict @@ -461,7 +466,7 @@ class Config: server_templates_dict[template_name] = ServerTemplate( template_name, stress_server_cmd, temp_dict['wrapperScriptPath'], temp_dict['serverPort'], temp_dict['serverArgs'].copy(), - temp_dict.get('willRunForever', 1)) + temp_dict.get('willRunForever', 1), temp_dict.get('env', {}).copy()) return server_templates_dict