mirror of https://github.com/grpc/grpc.git
Yuxuan Li
8 years ago
commit
c03858badf
52 changed files with 56 additions and 4794 deletions
@ -1,42 +0,0 @@ |
|||||||
%YAML 1.2 |
|
||||||
--- | |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
<%include file="../../apt_get_basic.include"/> |
|
||||||
<%include file="../../python_deps.include"/> |
|
||||||
<%include file="../../ccache_setup.include"/> |
|
||||||
<%include file="../../cxx_deps.include"/> |
|
||||||
<%include file="../../gcp_api_libraries.include"/> |
|
||||||
<%include file="../../csharp_deps.include"/> |
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
|
|
||||||
@ -1,41 +0,0 @@ |
|||||||
%YAML 1.2 |
|
||||||
--- | |
|
||||||
# Copyright 2015-2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
<%include file="../../apt_get_basic.include"/> |
|
||||||
<%include file="../../python_deps.include"/> |
|
||||||
<%include file="../../ccache_setup.include"/> |
|
||||||
<%include file="../../cxx_deps.include"/> |
|
||||||
<%include file="../../gcp_api_libraries.include"/> |
|
||||||
<%include file="../../clang_update.include"/> |
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
@ -1,41 +0,0 @@ |
|||||||
%YAML 1.2 |
|
||||||
--- | |
|
||||||
# Copyright 2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
<%include file="../../apt_get_basic.include"/> |
|
||||||
<%include file="../../python_deps.include"/> |
|
||||||
<%include file="../../ccache_setup.include"/> |
|
||||||
<%include file="../../cxx_deps.include"/> |
|
||||||
<%include file="../../gcp_api_libraries.include"/> |
|
||||||
<%include file="../../java_deps.include"/> |
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
@ -1,46 +0,0 @@ |
|||||||
%YAML 1.2 |
|
||||||
--- | |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
<%include file="../../apt_get_basic.include"/> |
|
||||||
<%include file="../../python_deps.include"/> |
|
||||||
<%include file="../../ruby_deps.include"/> |
|
||||||
<%include file="../../gcp_api_libraries.include"/> |
|
||||||
<%include file="../../php_deps.include"/> |
|
||||||
<%include file="../../run_tests_addons.include"/> |
|
||||||
# Install composer |
|
||||||
RUN curl -sS https://getcomposer.org/installer | php |
|
||||||
RUN mv composer.phar /usr/local/bin/composer |
|
||||||
|
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
|
|
||||||
@ -1,45 +0,0 @@ |
|||||||
%YAML 1.2 |
|
||||||
--- | |
|
||||||
# Copyright 2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
<%include file="../../apt_get_basic.include"/> |
|
||||||
<%include file="../../ccache_setup.include"/> |
|
||||||
<%include file="../../cxx_deps.include"/> |
|
||||||
<%include file="../../gcp_api_libraries.include"/> |
|
||||||
<%include file="../../python_deps.include"/> |
|
||||||
|
|
||||||
RUN pip install coverage |
|
||||||
RUN pip install oauth2client |
|
||||||
|
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
|
|
||||||
@ -1,42 +0,0 @@ |
|||||||
%YAML 1.2 |
|
||||||
--- | |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
<%include file="../../apt_get_basic.include"/> |
|
||||||
<%include file="../../python_deps.include"/> |
|
||||||
<%include file="../../ccache_setup.include"/> |
|
||||||
<%include file="../../cxx_deps.include"/> |
|
||||||
<%include file="../../gcp_api_libraries.include"/> |
|
||||||
<%include file="../../ruby_deps.include"/> |
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
|
|
||||||
@ -1,117 +0,0 @@ |
|||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
# Install Git and basic packages. |
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
autoconf \ |
|
||||||
autotools-dev \ |
|
||||||
build-essential \ |
|
||||||
bzip2 \ |
|
||||||
ccache \ |
|
||||||
curl \ |
|
||||||
gcc \ |
|
||||||
gcc-multilib \ |
|
||||||
git \ |
|
||||||
golang \ |
|
||||||
gyp \ |
|
||||||
lcov \ |
|
||||||
libc6 \ |
|
||||||
libc6-dbg \ |
|
||||||
libc6-dev \ |
|
||||||
libgtest-dev \ |
|
||||||
libtool \ |
|
||||||
make \ |
|
||||||
perl \ |
|
||||||
strace \ |
|
||||||
python-dev \ |
|
||||||
python-setuptools \ |
|
||||||
python-yaml \ |
|
||||||
telnet \ |
|
||||||
unzip \ |
|
||||||
wget \ |
|
||||||
zip && apt-get clean |
|
||||||
|
|
||||||
#================ |
|
||||||
# Build profiling |
|
||||||
RUN apt-get update && apt-get install -y time && apt-get clean |
|
||||||
|
|
||||||
#==================== |
|
||||||
# Python dependencies |
|
||||||
|
|
||||||
# Install dependencies |
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
python-all-dev \ |
|
||||||
python3-all-dev \ |
|
||||||
python-pip |
|
||||||
|
|
||||||
# Install Python packages from PyPI |
|
||||||
RUN pip install pip --upgrade |
|
||||||
RUN pip install virtualenv |
|
||||||
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 |
|
||||||
|
|
||||||
# Prepare ccache |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/g++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/cc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/c++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ |
|
||||||
|
|
||||||
#================= |
|
||||||
# C++ dependencies |
|
||||||
RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean |
|
||||||
|
|
||||||
# Google Cloud platform API libraries |
|
||||||
RUN apt-get update && apt-get install -y python-pip && apt-get clean |
|
||||||
RUN pip install --upgrade google-api-python-client |
|
||||||
|
|
||||||
|
|
||||||
#================ |
|
||||||
# C# dependencies |
|
||||||
|
|
||||||
# Update to a newer version of mono |
|
||||||
RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF |
|
||||||
RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list |
|
||||||
RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list |
|
||||||
RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list |
|
||||||
|
|
||||||
# Install dependencies |
|
||||||
RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ |
|
||||||
mono-devel \ |
|
||||||
ca-certificates-mono \ |
|
||||||
nuget \ |
|
||||||
&& apt-get clean |
|
||||||
|
|
||||||
RUN nuget update -self |
|
||||||
|
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
@ -1,51 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
# |
|
||||||
# Builds C# interop server and client in a base image. |
|
||||||
set -e |
|
||||||
|
|
||||||
mkdir -p /var/local/git |
|
||||||
git clone /var/local/jenkins/grpc /var/local/git/grpc |
|
||||||
# clone gRPC submodules, use data from locally cloned submodules where possible |
|
||||||
(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ |
|
||||||
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ |
|
||||||
${name}') |
|
||||||
|
|
||||||
# Copy service account keys if available |
|
||||||
cp -r /var/local/jenkins/service_account $HOME || true |
|
||||||
|
|
||||||
cd /var/local/git/grpc |
|
||||||
|
|
||||||
# Build C++ metrics client (to query the metrics from csharp stress client) |
|
||||||
make metrics_client -j |
|
||||||
|
|
||||||
# Build C# interop client & server |
|
||||||
tools/run_tests/run_tests.py -l csharp -c dbg --build_only |
|
||||||
|
|
||||||
@ -1,132 +0,0 @@ |
|||||||
# Copyright 2015-2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
# Install Git and basic packages. |
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
autoconf \ |
|
||||||
autotools-dev \ |
|
||||||
build-essential \ |
|
||||||
bzip2 \ |
|
||||||
ccache \ |
|
||||||
curl \ |
|
||||||
gcc \ |
|
||||||
gcc-multilib \ |
|
||||||
git \ |
|
||||||
golang \ |
|
||||||
gyp \ |
|
||||||
lcov \ |
|
||||||
libc6 \ |
|
||||||
libc6-dbg \ |
|
||||||
libc6-dev \ |
|
||||||
libgtest-dev \ |
|
||||||
libtool \ |
|
||||||
make \ |
|
||||||
perl \ |
|
||||||
strace \ |
|
||||||
python-dev \ |
|
||||||
python-setuptools \ |
|
||||||
python-yaml \ |
|
||||||
telnet \ |
|
||||||
unzip \ |
|
||||||
wget \ |
|
||||||
zip && apt-get clean |
|
||||||
|
|
||||||
#================ |
|
||||||
# Build profiling |
|
||||||
RUN apt-get update && apt-get install -y time && apt-get clean |
|
||||||
|
|
||||||
#==================== |
|
||||||
# Python dependencies |
|
||||||
|
|
||||||
# Install dependencies |
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
python-all-dev \ |
|
||||||
python3-all-dev \ |
|
||||||
python-pip |
|
||||||
|
|
||||||
# Install Python packages from PyPI |
|
||||||
RUN pip install pip --upgrade |
|
||||||
RUN pip install virtualenv |
|
||||||
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 |
|
||||||
|
|
||||||
# Prepare ccache |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/g++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/cc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/c++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ |
|
||||||
|
|
||||||
#================= |
|
||||||
# C++ dependencies |
|
||||||
RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean |
|
||||||
|
|
||||||
# Google Cloud platform API libraries |
|
||||||
RUN apt-get update && apt-get install -y python-pip && apt-get clean |
|
||||||
RUN pip install --upgrade google-api-python-client |
|
||||||
|
|
||||||
|
|
||||||
#================= |
|
||||||
# Update clang to a version with improved tsan and fuzzing capabilities |
|
||||||
|
|
||||||
RUN apt-get update && apt-get -y install python cmake && apt-get clean |
|
||||||
|
|
||||||
RUN git clone -n -b release_38 http://llvm.org/git/llvm.git && \ |
|
||||||
cd llvm && git checkout ad57503 && cd .. |
|
||||||
RUN git clone -n -b release_38 http://llvm.org/git/clang.git && \ |
|
||||||
cd clang && git checkout ad2c56e && cd .. |
|
||||||
RUN git clone -n -b release_38 http://llvm.org/git/compiler-rt.git && \ |
|
||||||
cd compiler-rt && git checkout 3176922 && cd .. |
|
||||||
RUN git clone -n -b release_38 \ |
|
||||||
http://llvm.org/git/clang-tools-extra.git && cd clang-tools-extra && \ |
|
||||||
git checkout c288525 && cd .. |
|
||||||
RUN git clone -n -b release_38 http://llvm.org/git/libcxx.git && \ |
|
||||||
cd libcxx && git checkout fda3549 && cd .. |
|
||||||
RUN git clone -n -b release_38 http://llvm.org/git/libcxxabi.git && \ |
|
||||||
cd libcxxabi && git checkout 8d4e51d && cd .. |
|
||||||
|
|
||||||
RUN mv clang llvm/tools |
|
||||||
RUN mv compiler-rt llvm/projects |
|
||||||
RUN mv clang-tools-extra llvm/tools/clang/tools |
|
||||||
RUN mv libcxx llvm/projects |
|
||||||
RUN mv libcxxabi llvm/projects |
|
||||||
|
|
||||||
RUN mkdir llvm-build |
|
||||||
RUN cd llvm-build && cmake \ |
|
||||||
-DCMAKE_BUILD_TYPE:STRING=Release \ |
|
||||||
-DCMAKE_INSTALL_PREFIX:STRING=/usr \ |
|
||||||
-DLLVM_TARGETS_TO_BUILD:STRING=X86 \ |
|
||||||
../llvm |
|
||||||
RUN make -C llvm-build -j 12 && make -C llvm-build install && rm -rf llvm-build |
|
||||||
|
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
@ -1,51 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
# |
|
||||||
# Builds C++ interop server and client in a base image. |
|
||||||
set -e |
|
||||||
|
|
||||||
mkdir -p /var/local/git |
|
||||||
git clone /var/local/jenkins/grpc /var/local/git/grpc |
|
||||||
# clone gRPC submodules, use data from locally cloned submodules where possible |
|
||||||
(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ |
|
||||||
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ |
|
||||||
${name}') |
|
||||||
|
|
||||||
# copy service account keys if available |
|
||||||
cp -r /var/local/jenkins/service_account $HOME || true |
|
||||||
|
|
||||||
cd /var/local/git/grpc |
|
||||||
|
|
||||||
make install-certs |
|
||||||
|
|
||||||
BUILD_TYPE=${BUILD_TYPE:=opt} |
|
||||||
|
|
||||||
# build C++ interop stress client, interop client and server |
|
||||||
make CONFIG=$BUILD_TYPE stress_test metrics_client interop_client interop_server |
|
||||||
@ -1,62 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
# |
|
||||||
# Builds Go interop server, Stress client and metrics client in a base image. |
|
||||||
set -e |
|
||||||
|
|
||||||
# Clone just the grpc-go source code without any dependencies. |
|
||||||
# We are cloning from a local git repo that contains the right revision |
|
||||||
# to test instead of using "go get" to download from Github directly. |
|
||||||
git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc |
|
||||||
|
|
||||||
# Clone the 'grpc' repo. We just need this for the wrapper scripts under |
|
||||||
# grpc/tools/gcp/stress_tests |
|
||||||
git clone /var/local/jenkins/grpc /var/local/git/grpc |
|
||||||
# clone gRPC submodules, use data from locally cloned submodules where possible |
|
||||||
(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ |
|
||||||
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ |
|
||||||
${name}') |
|
||||||
|
|
||||||
# copy service account keys if available |
|
||||||
cp -r /var/local/jenkins/service_account $HOME || true |
|
||||||
|
|
||||||
# Get dependencies from GitHub |
|
||||||
# NOTE: once grpc-go dependencies change, this needs to be updated manually |
|
||||||
# but we don't expect this to happen any time soon. |
|
||||||
go get github.com/golang/protobuf/proto |
|
||||||
go get golang.org/x/net/context |
|
||||||
go get golang.org/x/net/trace |
|
||||||
go get golang.org/x/oauth2 |
|
||||||
go get google.golang.org/cloud |
|
||||||
|
|
||||||
# Build the interop server, stress client and stress metrics client |
|
||||||
(cd src/google.golang.org/grpc/interop/server && go install) |
|
||||||
(cd src/google.golang.org/grpc/stress/client && go install) |
|
||||||
(cd src/google.golang.org/grpc/stress/metrics_client && go install) |
|
||||||
@ -1,117 +0,0 @@ |
|||||||
# Copyright 2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
# Install Git and basic packages. |
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
autoconf \ |
|
||||||
autotools-dev \ |
|
||||||
build-essential \ |
|
||||||
bzip2 \ |
|
||||||
ccache \ |
|
||||||
curl \ |
|
||||||
gcc \ |
|
||||||
gcc-multilib \ |
|
||||||
git \ |
|
||||||
golang \ |
|
||||||
gyp \ |
|
||||||
lcov \ |
|
||||||
libc6 \ |
|
||||||
libc6-dbg \ |
|
||||||
libc6-dev \ |
|
||||||
libgtest-dev \ |
|
||||||
libtool \ |
|
||||||
make \ |
|
||||||
perl \ |
|
||||||
strace \ |
|
||||||
python-dev \ |
|
||||||
python-setuptools \ |
|
||||||
python-yaml \ |
|
||||||
telnet \ |
|
||||||
unzip \ |
|
||||||
wget \ |
|
||||||
zip && apt-get clean |
|
||||||
|
|
||||||
#================ |
|
||||||
# Build profiling |
|
||||||
RUN apt-get update && apt-get install -y time && apt-get clean |
|
||||||
|
|
||||||
#==================== |
|
||||||
# Python dependencies |
|
||||||
|
|
||||||
# Install dependencies |
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
python-all-dev \ |
|
||||||
python3-all-dev \ |
|
||||||
python-pip |
|
||||||
|
|
||||||
# Install Python packages from PyPI |
|
||||||
RUN pip install pip --upgrade |
|
||||||
RUN pip install virtualenv |
|
||||||
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 |
|
||||||
|
|
||||||
# Prepare ccache |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/g++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/cc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/c++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ |
|
||||||
|
|
||||||
#================= |
|
||||||
# C++ dependencies |
|
||||||
RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean |
|
||||||
|
|
||||||
# Google Cloud platform API libraries |
|
||||||
RUN apt-get update && apt-get install -y python-pip && apt-get clean |
|
||||||
RUN pip install --upgrade google-api-python-client |
|
||||||
|
|
||||||
|
|
||||||
# Install JDK 8 and Git |
|
||||||
# |
|
||||||
RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \ |
|
||||||
echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list && \ |
|
||||||
echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && \ |
|
||||||
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 |
|
||||||
|
|
||||||
RUN apt-get update && apt-get -y install \ |
|
||||||
git \ |
|
||||||
libapr1 \ |
|
||||||
oracle-java8-installer \ |
|
||||||
&& \ |
|
||||||
apt-get clean && rm -r /var/cache/oracle-jdk8-installer/ |
|
||||||
|
|
||||||
ENV JAVA_HOME /usr/lib/jvm/java-8-oracle |
|
||||||
ENV PATH $PATH:$JAVA_HOME/bin |
|
||||||
|
|
||||||
|
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
@ -1,55 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
# |
|
||||||
# Builds C++ interop server and client in a base image. |
|
||||||
set -e |
|
||||||
|
|
||||||
mkdir -p /var/local/git |
|
||||||
# grpc-java repo |
|
||||||
git clone --recursive --depth 1 /var/local/jenkins/grpc-java /var/local/git/grpc-java |
|
||||||
|
|
||||||
# grpc repo (for metrics client and for the stress test wrapper scripts) |
|
||||||
git clone /var/local/jenkins/grpc /var/local/git/grpc |
|
||||||
# clone gRPC submodules, use data from locally cloned submodules where possible |
|
||||||
(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ |
|
||||||
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ |
|
||||||
${name}') |
|
||||||
|
|
||||||
# Copy service account keys if available |
|
||||||
cp -r /var/local/jenkins/service_account $HOME || true |
|
||||||
|
|
||||||
# First build the metrics client in grpc repo |
|
||||||
cd /var/local/git/grpc |
|
||||||
make metrics_client |
|
||||||
|
|
||||||
# Build all interop test targets (which includes interop server and stress test |
|
||||||
# client) in grpc-java repo |
|
||||||
cd /var/local/git/grpc-java |
|
||||||
./gradlew :grpc-interop-testing:installDist -PskipCodegen=true |
|
||||||
@ -1,109 +0,0 @@ |
|||||||
# Copyright 2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
# Install Git and basic packages. |
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
autoconf \ |
|
||||||
autotools-dev \ |
|
||||||
build-essential \ |
|
||||||
bzip2 \ |
|
||||||
ccache \ |
|
||||||
curl \ |
|
||||||
gcc \ |
|
||||||
gcc-multilib \ |
|
||||||
git \ |
|
||||||
golang \ |
|
||||||
gyp \ |
|
||||||
lcov \ |
|
||||||
libc6 \ |
|
||||||
libc6-dbg \ |
|
||||||
libc6-dev \ |
|
||||||
libgtest-dev \ |
|
||||||
libtool \ |
|
||||||
make \ |
|
||||||
perl \ |
|
||||||
strace \ |
|
||||||
python-dev \ |
|
||||||
python-setuptools \ |
|
||||||
python-yaml \ |
|
||||||
telnet \ |
|
||||||
unzip \ |
|
||||||
wget \ |
|
||||||
zip && apt-get clean |
|
||||||
|
|
||||||
#================ |
|
||||||
# Build profiling |
|
||||||
RUN apt-get update && apt-get install -y time && apt-get clean |
|
||||||
|
|
||||||
#==================== |
|
||||||
# Python dependencies |
|
||||||
|
|
||||||
# Install dependencies |
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
python-all-dev \ |
|
||||||
python3-all-dev \ |
|
||||||
python-pip |
|
||||||
|
|
||||||
# Install Python packages from PyPI |
|
||||||
RUN pip install pip --upgrade |
|
||||||
RUN pip install virtualenv |
|
||||||
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 |
|
||||||
|
|
||||||
#================== |
|
||||||
# Node dependencies |
|
||||||
|
|
||||||
# Install nvm |
|
||||||
RUN touch .profile |
|
||||||
RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash |
|
||||||
# Install all versions of node that we want to test |
|
||||||
RUN /bin/bash -l -c "nvm install 0.12 && npm config set cache /tmp/npm-cache" |
|
||||||
RUN /bin/bash -l -c "nvm install 4 && npm config set cache /tmp/npm-cache" |
|
||||||
RUN /bin/bash -l -c "nvm install 5 && npm config set cache /tmp/npm-cache" |
|
||||||
RUN /bin/bash -l -c "nvm alias default 4" |
|
||||||
# Google Cloud platform API libraries |
|
||||||
RUN apt-get update && apt-get install -y python-pip && apt-get clean |
|
||||||
RUN pip install --upgrade google-api-python-client |
|
||||||
|
|
||||||
|
|
||||||
# Prepare ccache |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/g++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/cc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/c++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ |
|
||||||
|
|
||||||
|
|
||||||
RUN mkdir /var/local/jenkins |
|
||||||
|
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
@ -1,48 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
# |
|
||||||
# Builds Node interop server and client in a base image. |
|
||||||
set -e |
|
||||||
|
|
||||||
mkdir -p /var/local/git |
|
||||||
git clone /var/local/jenkins/grpc /var/local/git/grpc |
|
||||||
# clone gRPC submodules, use data from locally cloned submodules where possible |
|
||||||
(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ |
|
||||||
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ |
|
||||||
${name}') |
|
||||||
|
|
||||||
# copy service account keys if available |
|
||||||
cp -r /var/local/jenkins/service_account $HOME || true |
|
||||||
|
|
||||||
cd /var/local/git/grpc |
|
||||||
|
|
||||||
# build Node interop client & server |
|
||||||
npm install -g node-gyp |
|
||||||
npm install --unsafe-perm --build-from-source |
|
||||||
@ -1,125 +0,0 @@ |
|||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
# Install Git and basic packages. |
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
autoconf \ |
|
||||||
autotools-dev \ |
|
||||||
build-essential \ |
|
||||||
bzip2 \ |
|
||||||
ccache \ |
|
||||||
curl \ |
|
||||||
gcc \ |
|
||||||
gcc-multilib \ |
|
||||||
git \ |
|
||||||
golang \ |
|
||||||
gyp \ |
|
||||||
lcov \ |
|
||||||
libc6 \ |
|
||||||
libc6-dbg \ |
|
||||||
libc6-dev \ |
|
||||||
libgtest-dev \ |
|
||||||
libtool \ |
|
||||||
make \ |
|
||||||
perl \ |
|
||||||
strace \ |
|
||||||
python-dev \ |
|
||||||
python-setuptools \ |
|
||||||
python-yaml \ |
|
||||||
telnet \ |
|
||||||
unzip \ |
|
||||||
wget \ |
|
||||||
zip && apt-get clean |
|
||||||
|
|
||||||
#================ |
|
||||||
# Build profiling |
|
||||||
RUN apt-get update && apt-get install -y time && apt-get clean |
|
||||||
|
|
||||||
#==================== |
|
||||||
# Python dependencies |
|
||||||
|
|
||||||
# Install dependencies |
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
python-all-dev \ |
|
||||||
python3-all-dev \ |
|
||||||
python-pip |
|
||||||
|
|
||||||
# Install Python packages from PyPI |
|
||||||
RUN pip install pip --upgrade |
|
||||||
RUN pip install virtualenv |
|
||||||
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 |
|
||||||
|
|
||||||
#================== |
|
||||||
# Ruby dependencies |
|
||||||
|
|
||||||
# Install rvm |
|
||||||
RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 |
|
||||||
RUN \curl -sSL https://get.rvm.io | bash -s stable |
|
||||||
|
|
||||||
# Install Ruby 2.1 |
|
||||||
RUN /bin/bash -l -c "rvm install ruby-2.1" |
|
||||||
RUN /bin/bash -l -c "rvm use --default ruby-2.1" |
|
||||||
RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" |
|
||||||
RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" |
|
||||||
RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" |
|
||||||
RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" |
|
||||||
|
|
||||||
# Google Cloud platform API libraries |
|
||||||
RUN apt-get update && apt-get install -y python-pip && apt-get clean |
|
||||||
RUN pip install --upgrade google-api-python-client |
|
||||||
|
|
||||||
|
|
||||||
#================= |
|
||||||
# PHP dependencies |
|
||||||
|
|
||||||
# Install dependencies |
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
git php5 php5-dev phpunit unzip |
|
||||||
|
|
||||||
# Prepare ccache |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/g++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/cc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/c++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ |
|
||||||
|
|
||||||
|
|
||||||
RUN mkdir /var/local/jenkins |
|
||||||
|
|
||||||
# Install composer |
|
||||||
RUN curl -sS https://getcomposer.org/installer | php |
|
||||||
RUN mv composer.phar /usr/local/bin/composer |
|
||||||
|
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
@ -1,57 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
# |
|
||||||
# Builds PHP interop server and client in a base image. |
|
||||||
set -ex |
|
||||||
|
|
||||||
mkdir -p /var/local/git |
|
||||||
git clone /var/local/jenkins/grpc /var/local/git/grpc |
|
||||||
# clone gRPC submodules, use data from locally cloned submodules where possible |
|
||||||
(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ |
|
||||||
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ |
|
||||||
${name}') |
|
||||||
|
|
||||||
# copy service account keys if available |
|
||||||
cp -r /var/local/jenkins/service_account $HOME || true |
|
||||||
|
|
||||||
cd /var/local/git/grpc |
|
||||||
|
|
||||||
make install-certs |
|
||||||
|
|
||||||
# gRPC core and protobuf need to be installed |
|
||||||
make install |
|
||||||
|
|
||||||
(cd src/php/ext/grpc && phpize && ./configure && make) |
|
||||||
|
|
||||||
(cd third_party/protobuf && make install) |
|
||||||
|
|
||||||
(cd src/php && php -d extension=ext/grpc/modules/grpc.so /usr/local/bin/composer install) |
|
||||||
|
|
||||||
(cd src/php && ./bin/generate_proto_php.sh) |
|
||||||
@ -1,103 +0,0 @@ |
|||||||
# Copyright 2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
# Install Git and basic packages. |
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
autoconf \ |
|
||||||
autotools-dev \ |
|
||||||
build-essential \ |
|
||||||
bzip2 \ |
|
||||||
ccache \ |
|
||||||
curl \ |
|
||||||
gcc \ |
|
||||||
gcc-multilib \ |
|
||||||
git \ |
|
||||||
golang \ |
|
||||||
gyp \ |
|
||||||
lcov \ |
|
||||||
libc6 \ |
|
||||||
libc6-dbg \ |
|
||||||
libc6-dev \ |
|
||||||
libgtest-dev \ |
|
||||||
libtool \ |
|
||||||
make \ |
|
||||||
perl \ |
|
||||||
strace \ |
|
||||||
python-dev \ |
|
||||||
python-setuptools \ |
|
||||||
python-yaml \ |
|
||||||
telnet \ |
|
||||||
unzip \ |
|
||||||
wget \ |
|
||||||
zip && apt-get clean |
|
||||||
|
|
||||||
#================ |
|
||||||
# Build profiling |
|
||||||
RUN apt-get update && apt-get install -y time && apt-get clean |
|
||||||
|
|
||||||
# Prepare ccache |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/g++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/cc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/c++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ |
|
||||||
|
|
||||||
#================= |
|
||||||
# C++ dependencies |
|
||||||
RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean |
|
||||||
|
|
||||||
# Google Cloud platform API libraries |
|
||||||
RUN apt-get update && apt-get install -y python-pip && apt-get clean |
|
||||||
RUN pip install --upgrade google-api-python-client |
|
||||||
|
|
||||||
|
|
||||||
#==================== |
|
||||||
# Python dependencies |
|
||||||
|
|
||||||
# Install dependencies |
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
python-all-dev \ |
|
||||||
python3-all-dev \ |
|
||||||
python-pip |
|
||||||
|
|
||||||
# Install Python packages from PyPI |
|
||||||
RUN pip install pip --upgrade |
|
||||||
RUN pip install virtualenv |
|
||||||
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 |
|
||||||
|
|
||||||
|
|
||||||
RUN pip install coverage |
|
||||||
RUN pip install oauth2client |
|
||||||
|
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
@ -1,49 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
# |
|
||||||
# Builds Python interop server and client in a base image. |
|
||||||
set -e |
|
||||||
|
|
||||||
mkdir -p /var/local/git |
|
||||||
git clone /var/local/jenkins/grpc /var/local/git/grpc |
|
||||||
# clone gRPC submodules, use data from locally cloned submodules where possible |
|
||||||
(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ |
|
||||||
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ |
|
||||||
${name}') |
|
||||||
|
|
||||||
# copy service account keys if available |
|
||||||
cp -r /var/local/jenkins/service_account $HOME || true |
|
||||||
|
|
||||||
cd /var/local/git/grpc |
|
||||||
tools/run_tests/run_tests.py -l python -c opt --build_only |
|
||||||
|
|
||||||
# Build c++ interop client |
|
||||||
make metrics_client -j |
|
||||||
|
|
||||||
@ -1,114 +0,0 @@ |
|||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
FROM debian:jessie |
|
||||||
|
|
||||||
# Install Git and basic packages. |
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
autoconf \ |
|
||||||
autotools-dev \ |
|
||||||
build-essential \ |
|
||||||
bzip2 \ |
|
||||||
ccache \ |
|
||||||
curl \ |
|
||||||
gcc \ |
|
||||||
gcc-multilib \ |
|
||||||
git \ |
|
||||||
golang \ |
|
||||||
gyp \ |
|
||||||
lcov \ |
|
||||||
libc6 \ |
|
||||||
libc6-dbg \ |
|
||||||
libc6-dev \ |
|
||||||
libgtest-dev \ |
|
||||||
libtool \ |
|
||||||
make \ |
|
||||||
perl \ |
|
||||||
strace \ |
|
||||||
python-dev \ |
|
||||||
python-setuptools \ |
|
||||||
python-yaml \ |
|
||||||
telnet \ |
|
||||||
unzip \ |
|
||||||
wget \ |
|
||||||
zip && apt-get clean |
|
||||||
|
|
||||||
#================ |
|
||||||
# Build profiling |
|
||||||
RUN apt-get update && apt-get install -y time && apt-get clean |
|
||||||
|
|
||||||
#==================== |
|
||||||
# Python dependencies |
|
||||||
|
|
||||||
# Install dependencies |
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \ |
|
||||||
python-all-dev \ |
|
||||||
python3-all-dev \ |
|
||||||
python-pip |
|
||||||
|
|
||||||
# Install Python packages from PyPI |
|
||||||
RUN pip install pip --upgrade |
|
||||||
RUN pip install virtualenv |
|
||||||
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0 |
|
||||||
|
|
||||||
# Prepare ccache |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/g++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/cc |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/c++ |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang |
|
||||||
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ |
|
||||||
|
|
||||||
#================= |
|
||||||
# C++ dependencies |
|
||||||
RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean |
|
||||||
|
|
||||||
# Google Cloud platform API libraries |
|
||||||
RUN apt-get update && apt-get install -y python-pip && apt-get clean |
|
||||||
RUN pip install --upgrade google-api-python-client |
|
||||||
|
|
||||||
|
|
||||||
#================== |
|
||||||
# Ruby dependencies |
|
||||||
|
|
||||||
# Install rvm |
|
||||||
RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 |
|
||||||
RUN \curl -sSL https://get.rvm.io | bash -s stable |
|
||||||
|
|
||||||
# Install Ruby 2.1 |
|
||||||
RUN /bin/bash -l -c "rvm install ruby-2.1" |
|
||||||
RUN /bin/bash -l -c "rvm use --default ruby-2.1" |
|
||||||
RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" |
|
||||||
RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" |
|
||||||
RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" |
|
||||||
RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" |
|
||||||
|
|
||||||
# Define the default command. |
|
||||||
CMD ["bash"] |
|
||||||
@ -1,52 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
# |
|
||||||
# Builds Ruby interop server and client in a base image. |
|
||||||
set -e |
|
||||||
|
|
||||||
mkdir -p /var/local/git |
|
||||||
git clone /var/local/jenkins/grpc /var/local/git/grpc |
|
||||||
# clone gRPC submodules, use data from locally cloned submodules where possible |
|
||||||
(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ |
|
||||||
&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ |
|
||||||
${name}') |
|
||||||
|
|
||||||
# Copy service account keys if available |
|
||||||
cp -r /var/local/jenkins/service_account $HOME || true |
|
||||||
|
|
||||||
cd /var/local/git/grpc |
|
||||||
rvm --default use ruby-2.1 |
|
||||||
|
|
||||||
# Build Ruby interop client and server |
|
||||||
(cd src/ruby && gem update bundler && bundle && rake compile) |
|
||||||
|
|
||||||
# Build c++ metrics client to query the metrics from ruby stress client |
|
||||||
make metrics_client -j |
|
||||||
|
|
||||||
@ -1,206 +0,0 @@ |
|||||||
#!/usr/bin/env python2.7 |
|
||||||
# Copyright 2015-2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
import datetime |
|
||||||
import os |
|
||||||
import re |
|
||||||
import resource |
|
||||||
import select |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
import time |
|
||||||
|
|
||||||
from stress_test_utils import EventType |
|
||||||
from stress_test_utils import BigQueryHelper |
|
||||||
|
|
||||||
|
|
||||||
# TODO (sree): Write a python grpc client to directly query the metrics instead |
|
||||||
# of calling metrics_client |
|
||||||
def _get_qps(metrics_cmd): |
|
||||||
qps = 0 |
|
||||||
try: |
|
||||||
# Note: gpr_log() writes even non-error messages to stderr stream. So it is |
|
||||||
# important that we set stderr=subprocess.STDOUT |
|
||||||
p = subprocess.Popen(args=metrics_cmd, |
|
||||||
stdout=subprocess.PIPE, |
|
||||||
stderr=subprocess.STDOUT) |
|
||||||
retcode = p.wait() |
|
||||||
(out_str, err_str) = p.communicate() |
|
||||||
if retcode != 0: |
|
||||||
print 'Error in reading metrics information' |
|
||||||
print 'Output: ', out_str |
|
||||||
else: |
|
||||||
# The overall qps is printed at the end of the line |
|
||||||
m = re.search('\d+$', out_str) |
|
||||||
qps = int(m.group()) if m else 0 |
|
||||||
except Exception as ex: |
|
||||||
print 'Exception while reading metrics information: ' + str(ex) |
|
||||||
return qps |
|
||||||
|
|
||||||
|
|
||||||
def run_client(): |
|
||||||
"""This is a wrapper around the stress test client and performs the following: |
|
||||||
1) Create the following two tables in Big Query: |
|
||||||
(i) Summary table: To record events like the test started, completed |
|
||||||
successfully or failed |
|
||||||
(ii) Qps table: To periodically record the QPS sent by this client |
|
||||||
2) Start the stress test client and add a row in the Big Query summary |
|
||||||
table |
|
||||||
3) Once every few seconds (as specificed by the poll_interval_secs) poll |
|
||||||
the status of the stress test client process and perform the |
|
||||||
following: |
|
||||||
3.1) If the process is still running, get the current qps by invoking |
|
||||||
the metrics client program and add a row in the Big Query |
|
||||||
Qps table. Sleep for a duration specified by poll_interval_secs |
|
||||||
3.2) If the process exited successfully, add a row in the Big Query |
|
||||||
Summary table and exit |
|
||||||
3.3) If the process failed, add a row in Big Query summary table and |
|
||||||
wait forever. |
|
||||||
NOTE: This script typically runs inside a GKE pod which means |
|
||||||
that the pod gets destroyed when the script exits. However, in |
|
||||||
case the stress test client fails, we would not want the pod to |
|
||||||
be destroyed (since we might want to connect to the pod for |
|
||||||
examining logs). This is the reason why the script waits forever |
|
||||||
in case of failures |
|
||||||
""" |
|
||||||
# Set the 'core file' size to 'unlimited' so that 'core' files are generated |
|
||||||
# if the client crashes (Note: This is not relevant for Java and Go clients) |
|
||||||
resource.setrlimit(resource.RLIMIT_CORE, |
|
||||||
(resource.RLIM_INFINITY, resource.RLIM_INFINITY)) |
|
||||||
|
|
||||||
env = dict(os.environ) |
|
||||||
image_type = env['STRESS_TEST_IMAGE_TYPE'] |
|
||||||
stress_client_cmd = env['STRESS_TEST_CMD'].split() |
|
||||||
args_str = env['STRESS_TEST_ARGS_STR'] |
|
||||||
metrics_client_cmd = env['METRICS_CLIENT_CMD'].split() |
|
||||||
metrics_client_args_str = env['METRICS_CLIENT_ARGS_STR'] |
|
||||||
run_id = env['RUN_ID'] |
|
||||||
pod_name = env['POD_NAME'] |
|
||||||
logfile_name = env.get('LOGFILE_NAME') |
|
||||||
poll_interval_secs = float(env['POLL_INTERVAL_SECS']) |
|
||||||
project_id = env['GCP_PROJECT_ID'] |
|
||||||
dataset_id = env['DATASET_ID'] |
|
||||||
summary_table_id = env['SUMMARY_TABLE_ID'] |
|
||||||
qps_table_id = env['QPS_TABLE_ID'] |
|
||||||
# The following parameter is to inform us whether the stress client runs |
|
||||||
# forever until forcefully stopped or will it naturally stop after sometime. |
|
||||||
# This way, we know that the stress client process should not terminate (even |
|
||||||
# if it does with a success exit code) and flag the termination as a failure |
|
||||||
will_run_forever = env.get('WILL_RUN_FOREVER', '1') |
|
||||||
|
|
||||||
bq_helper = BigQueryHelper(run_id, image_type, pod_name, project_id, |
|
||||||
dataset_id, summary_table_id, qps_table_id) |
|
||||||
bq_helper.initialize() |
|
||||||
|
|
||||||
# Create BigQuery Dataset and Tables: Summary Table and Metrics Table |
|
||||||
if not bq_helper.setup_tables(): |
|
||||||
print 'Error in creating BigQuery tables' |
|
||||||
return |
|
||||||
|
|
||||||
start_time = datetime.datetime.now() |
|
||||||
|
|
||||||
logfile = None |
|
||||||
details = 'Logging to stdout' |
|
||||||
if logfile_name is not None: |
|
||||||
print 'Opening logfile: %s ...' % logfile_name |
|
||||||
details = 'Logfile: %s' % logfile_name |
|
||||||
logfile = open(logfile_name, 'w') |
|
||||||
|
|
||||||
metrics_cmd = metrics_client_cmd + [x |
|
||||||
for x in metrics_client_args_str.split()] |
|
||||||
stress_cmd = stress_client_cmd + [x for x in args_str.split()] |
|
||||||
|
|
||||||
details = '%s, Metrics command: %s, Stress client command: %s' % ( |
|
||||||
details, str(metrics_cmd), str(stress_cmd)) |
|
||||||
# Update status that the test is starting (in the status table) |
|
||||||
bq_helper.insert_summary_row(EventType.STARTING, details) |
|
||||||
|
|
||||||
print 'Launching process %s ...' % stress_cmd |
|
||||||
stress_p = subprocess.Popen(args=stress_cmd, |
|
||||||
stdout=logfile, |
|
||||||
stderr=subprocess.STDOUT) |
|
||||||
|
|
||||||
qps_history = [1, 1, 1] # Maintain the last 3 qps readings |
|
||||||
qps_history_idx = 0 # Index into the qps_history list |
|
||||||
|
|
||||||
is_running_status_written = False |
|
||||||
is_error = False |
|
||||||
while True: |
|
||||||
# Check if stress_client is still running. If so, collect metrics and upload |
|
||||||
# to BigQuery status table |
|
||||||
# If stress_p.poll() is not None, it means that the stress client terminated |
|
||||||
if stress_p.poll() is not None: |
|
||||||
end_time = datetime.datetime.now().isoformat() |
|
||||||
event_type = EventType.SUCCESS |
|
||||||
details = 'End time: %s' % end_time |
|
||||||
if will_run_forever == '1' or stress_p.returncode != 0: |
|
||||||
event_type = EventType.FAILURE |
|
||||||
details = 'Return code = %d. End time: %s' % (stress_p.returncode, |
|
||||||
end_time) |
|
||||||
is_error = True |
|
||||||
bq_helper.insert_summary_row(event_type, details) |
|
||||||
print details |
|
||||||
break |
|
||||||
|
|
||||||
if not is_running_status_written: |
|
||||||
bq_helper.insert_summary_row(EventType.RUNNING, '') |
|
||||||
is_running_status_written = True |
|
||||||
|
|
||||||
# Stress client still running. Get metrics |
|
||||||
qps = _get_qps(metrics_cmd) |
|
||||||
qps_recorded_at = datetime.datetime.now().isoformat() |
|
||||||
print 'qps: %d at %s' % (qps, qps_recorded_at) |
|
||||||
|
|
||||||
# If QPS has been zero for the last 3 iterations, flag it as error and exit |
|
||||||
qps_history[qps_history_idx] = qps |
|
||||||
qps_history_idx = (qps_history_idx + 1) % len(qps_history) |
|
||||||
if sum(qps_history) == 0: |
|
||||||
details = 'QPS has been zero for the last %d seconds - as of : %s' % ( |
|
||||||
poll_interval_secs * 3, qps_recorded_at) |
|
||||||
is_error = True |
|
||||||
bq_helper.insert_summary_row(EventType.FAILURE, details) |
|
||||||
print details |
|
||||||
break |
|
||||||
|
|
||||||
# Upload qps metrics to BiqQuery |
|
||||||
bq_helper.insert_qps_row(qps, qps_recorded_at) |
|
||||||
|
|
||||||
time.sleep(poll_interval_secs) |
|
||||||
|
|
||||||
if is_error: |
|
||||||
print 'Waiting indefinitely..' |
|
||||||
select.select([], [], []) |
|
||||||
|
|
||||||
print 'Completed' |
|
||||||
return |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__': |
|
||||||
run_client() |
|
||||||
@ -1,37 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2015-2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
# This is a wrapper script that was created to help run_server.py and |
|
||||||
# run_client.py to launch 'node js' stress clients and stress servers |
|
||||||
source ~/.nvm/nvm.sh |
|
||||||
|
|
||||||
set -ex |
|
||||||
|
|
||||||
$@ |
|
||||||
@ -1,37 +0,0 @@ |
|||||||
#!/bin/bash |
|
||||||
# Copyright 2015-2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
# This is a wrapper script that was created to help run_server.py and |
|
||||||
# run_client.py to launch 'node js' stress clients and stress servers |
|
||||||
source /etc/profile.d/rvm.sh |
|
||||||
|
|
||||||
set -ex |
|
||||||
|
|
||||||
$@ |
|
||||||
@ -1,138 +0,0 @@ |
|||||||
#!/usr/bin/env python2.7 |
|
||||||
# Copyright 2015-2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
import datetime |
|
||||||
import os |
|
||||||
import resource |
|
||||||
import select |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
import time |
|
||||||
|
|
||||||
from stress_test_utils import BigQueryHelper |
|
||||||
from stress_test_utils import EventType |
|
||||||
|
|
||||||
|
|
||||||
def run_server(): |
|
||||||
"""This is a wrapper around the interop server and performs the following: |
|
||||||
1) Create a 'Summary table' in Big Query to record events like the server |
|
||||||
started, completed successfully or failed. NOTE: This also creates |
|
||||||
another table called the QPS table which is currently NOT needed on the |
|
||||||
server (it is needed on the stress test clients) |
|
||||||
2) Start the server process and add a row in Big Query summary table |
|
||||||
3) Wait for the server process to terminate. The server process does not |
|
||||||
terminate unless there is an error. |
|
||||||
If the server process terminated with a failure, add a row in Big Query |
|
||||||
and wait forever. |
|
||||||
NOTE: This script typically runs inside a GKE pod which means that the |
|
||||||
pod gets destroyed when the script exits. However, in case the server |
|
||||||
process fails, we would not want the pod to be destroyed (since we |
|
||||||
might want to connect to the pod for examining logs). This is the |
|
||||||
reason why the script waits forever in case of failures. |
|
||||||
""" |
|
||||||
# Set the 'core file' size to 'unlimited' so that 'core' files are generated |
|
||||||
# if the server crashes (Note: This is not relevant for Java and Go servers) |
|
||||||
resource.setrlimit(resource.RLIMIT_CORE, |
|
||||||
(resource.RLIM_INFINITY, resource.RLIM_INFINITY)) |
|
||||||
|
|
||||||
# Read the parameters from environment variables |
|
||||||
env = dict(os.environ) |
|
||||||
|
|
||||||
run_id = env['RUN_ID'] # The unique run id for this test |
|
||||||
image_type = env['STRESS_TEST_IMAGE_TYPE'] |
|
||||||
stress_server_cmd = env['STRESS_TEST_CMD'].split() |
|
||||||
args_str = env['STRESS_TEST_ARGS_STR'] |
|
||||||
pod_name = env['POD_NAME'] |
|
||||||
project_id = env['GCP_PROJECT_ID'] |
|
||||||
dataset_id = env['DATASET_ID'] |
|
||||||
summary_table_id = env['SUMMARY_TABLE_ID'] |
|
||||||
qps_table_id = env['QPS_TABLE_ID'] |
|
||||||
# The following parameter is to inform us whether the server runs forever |
|
||||||
# until forcefully stopped or will it naturally stop after sometime. |
|
||||||
# This way, we know that the process should not terminate (even if it does |
|
||||||
# with a success exit code) and flag any termination as a failure. |
|
||||||
will_run_forever = env.get('WILL_RUN_FOREVER', '1') |
|
||||||
|
|
||||||
logfile_name = env.get('LOGFILE_NAME') |
|
||||||
|
|
||||||
print('pod_name: %s, project_id: %s, run_id: %s, dataset_id: %s, ' |
|
||||||
'summary_table_id: %s, qps_table_id: %s') % (pod_name, project_id, |
|
||||||
run_id, dataset_id, |
|
||||||
summary_table_id, |
|
||||||
qps_table_id) |
|
||||||
|
|
||||||
bq_helper = BigQueryHelper(run_id, image_type, pod_name, project_id, |
|
||||||
dataset_id, summary_table_id, qps_table_id) |
|
||||||
bq_helper.initialize() |
|
||||||
|
|
||||||
# Create BigQuery Dataset and Tables: Summary Table and Metrics Table |
|
||||||
if not bq_helper.setup_tables(): |
|
||||||
print 'Error in creating BigQuery tables' |
|
||||||
return |
|
||||||
|
|
||||||
start_time = datetime.datetime.now() |
|
||||||
|
|
||||||
logfile = None |
|
||||||
details = 'Logging to stdout' |
|
||||||
if logfile_name is not None: |
|
||||||
print 'Opening log file: ', logfile_name |
|
||||||
logfile = open(logfile_name, 'w') |
|
||||||
details = 'Logfile: %s' % logfile_name |
|
||||||
|
|
||||||
stress_cmd = stress_server_cmd + [x for x in args_str.split()] |
|
||||||
|
|
||||||
details = '%s, Stress server command: %s' % (details, str(stress_cmd)) |
|
||||||
# Update status that the test is starting (in the status table) |
|
||||||
bq_helper.insert_summary_row(EventType.STARTING, details) |
|
||||||
|
|
||||||
print 'Launching process %s ...' % stress_cmd |
|
||||||
stress_p = subprocess.Popen(args=stress_cmd, |
|
||||||
stdout=logfile, |
|
||||||
stderr=subprocess.STDOUT) |
|
||||||
|
|
||||||
# Update the status to running if subprocess.Popen launched the server |
|
||||||
if stress_p.poll() is None: |
|
||||||
bq_helper.insert_summary_row(EventType.RUNNING, '') |
|
||||||
|
|
||||||
# Wait for the server process to terminate |
|
||||||
returncode = stress_p.wait() |
|
||||||
|
|
||||||
if will_run_forever == '1' or returncode != 0: |
|
||||||
end_time = datetime.datetime.now().isoformat() |
|
||||||
event_type = EventType.FAILURE |
|
||||||
details = 'Returncode: %d; End time: %s' % (returncode, end_time) |
|
||||||
bq_helper.insert_summary_row(event_type, details) |
|
||||||
print 'Waiting indefinitely..' |
|
||||||
select.select([], [], []) |
|
||||||
return returncode |
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__': |
|
||||||
run_server() |
|
||||||
@ -1,217 +0,0 @@ |
|||||||
#!/usr/bin/env python2.7 |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
import datetime |
|
||||||
import json |
|
||||||
import os |
|
||||||
import re |
|
||||||
import select |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
import time |
|
||||||
|
|
||||||
# Import big_query_utils module |
|
||||||
bq_utils_dir = os.path.abspath(os.path.join( |
|
||||||
os.path.dirname(__file__), '../utils')) |
|
||||||
sys.path.append(bq_utils_dir) |
|
||||||
import big_query_utils as bq_utils |
|
||||||
|
|
||||||
|
|
||||||
class EventType: |
|
||||||
STARTING = 'STARTING' |
|
||||||
RUNNING = 'RUNNING' |
|
||||||
SUCCESS = 'SUCCESS' |
|
||||||
FAILURE = 'FAILURE' |
|
||||||
|
|
||||||
|
|
||||||
class BigQueryHelper: |
|
||||||
"""Helper class for the stress test wrappers to interact with BigQuery. |
|
||||||
""" |
|
||||||
|
|
||||||
def __init__(self, run_id, image_type, pod_name, project_id, dataset_id, |
|
||||||
summary_table_id, qps_table_id): |
|
||||||
self.run_id = run_id |
|
||||||
self.image_type = image_type |
|
||||||
self.pod_name = pod_name |
|
||||||
self.project_id = project_id |
|
||||||
self.dataset_id = dataset_id |
|
||||||
self.summary_table_id = summary_table_id |
|
||||||
self.qps_table_id = qps_table_id |
|
||||||
|
|
||||||
def initialize(self): |
|
||||||
self.bq = bq_utils.create_big_query() |
|
||||||
|
|
||||||
def setup_tables(self): |
|
||||||
return bq_utils.create_dataset(self.bq, self.project_id, self.dataset_id) \ |
|
||||||
and self.__create_summary_table() \ |
|
||||||
and self.__create_qps_table() |
|
||||||
|
|
||||||
def insert_summary_row(self, event_type, details): |
|
||||||
row_values_dict = { |
|
||||||
'run_id': self.run_id, |
|
||||||
'image_type': self.image_type, |
|
||||||
'pod_name': self.pod_name, |
|
||||||
'event_date': datetime.datetime.now().isoformat(), |
|
||||||
'event_type': event_type, |
|
||||||
'details': details |
|
||||||
} |
|
||||||
# row_unique_id is something that uniquely identifies the row (BigQuery uses |
|
||||||
# it for duplicate detection). |
|
||||||
row_unique_id = '%s_%s_%s' % (self.run_id, self.pod_name, event_type) |
|
||||||
row = bq_utils.make_row(row_unique_id, row_values_dict) |
|
||||||
return bq_utils.insert_rows(self.bq, self.project_id, self.dataset_id, |
|
||||||
self.summary_table_id, [row]) |
|
||||||
|
|
||||||
def insert_qps_row(self, qps, recorded_at): |
|
||||||
row_values_dict = { |
|
||||||
'run_id': self.run_id, |
|
||||||
'pod_name': self.pod_name, |
|
||||||
'recorded_at': recorded_at, |
|
||||||
'qps': qps |
|
||||||
} |
|
||||||
|
|
||||||
# row_unique_id is something that uniquely identifies the row (BigQuery uses |
|
||||||
# it for duplicate detection). |
|
||||||
row_unique_id = '%s_%s_%s' % (self.run_id, self.pod_name, recorded_at) |
|
||||||
row = bq_utils.make_row(row_unique_id, row_values_dict) |
|
||||||
return bq_utils.insert_rows(self.bq, self.project_id, self.dataset_id, |
|
||||||
self.qps_table_id, [row]) |
|
||||||
|
|
||||||
def check_if_any_tests_failed(self, num_query_retries=3, timeout_msec=30000): |
|
||||||
query = ('SELECT event_type FROM %s.%s WHERE run_id = \'%s\' AND ' |
|
||||||
'event_type="%s"') % (self.dataset_id, self.summary_table_id, |
|
||||||
self.run_id, EventType.FAILURE) |
|
||||||
page = None |
|
||||||
try: |
|
||||||
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) |
|
||||||
job_id = query_job['jobReference']['jobId'] |
|
||||||
project_id = query_job['jobReference']['projectId'] |
|
||||||
page = self.bq.jobs().getQueryResults( |
|
||||||
projectId=project_id, |
|
||||||
jobId=job_id, |
|
||||||
timeoutMs=timeout_msec).execute(num_retries=num_query_retries) |
|
||||||
|
|
||||||
if not page['jobComplete']: |
|
||||||
print('TIMEOUT ERROR: The query %s timed out. Current timeout value is' |
|
||||||
' %d msec. Returning False (i.e assuming there are no failures)' |
|
||||||
) % (query, timeout_msec) |
|
||||||
return False |
|
||||||
|
|
||||||
num_failures = int(page['totalRows']) |
|
||||||
print 'num rows: ', num_failures |
|
||||||
return num_failures > 0 |
|
||||||
except: |
|
||||||
print 'Exception in check_if_any_tests_failed(). Info: ', sys.exc_info() |
|
||||||
print 'Query: ', query |
|
||||||
|
|
||||||
def print_summary_records(self, num_query_retries=3): |
|
||||||
line = '-' * 120 |
|
||||||
print line |
|
||||||
print 'Summary records' |
|
||||||
print 'Run Id: ', self.run_id |
|
||||||
print 'Dataset Id: ', self.dataset_id |
|
||||||
print line |
|
||||||
query = ('SELECT pod_name, image_type, event_type, event_date, details' |
|
||||||
' FROM %s.%s WHERE run_id = \'%s\' ORDER by event_date;') % ( |
|
||||||
self.dataset_id, self.summary_table_id, self.run_id) |
|
||||||
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) |
|
||||||
|
|
||||||
print '{:<25} {:<12} {:<12} {:<30} {}'.format('Pod name', 'Image type', |
|
||||||
'Event type', 'Date', |
|
||||||
'Details') |
|
||||||
print line |
|
||||||
page_token = None |
|
||||||
while True: |
|
||||||
page = self.bq.jobs().getQueryResults( |
|
||||||
pageToken=page_token, |
|
||||||
**query_job['jobReference']).execute(num_retries=num_query_retries) |
|
||||||
rows = page.get('rows', []) |
|
||||||
for row in rows: |
|
||||||
print '{:<25} {:<12} {:<12} {:<30} {}'.format(row['f'][0]['v'], |
|
||||||
row['f'][1]['v'], |
|
||||||
row['f'][2]['v'], |
|
||||||
row['f'][3]['v'], |
|
||||||
row['f'][4]['v']) |
|
||||||
page_token = page.get('pageToken') |
|
||||||
if not page_token: |
|
||||||
break |
|
||||||
|
|
||||||
def print_qps_records(self, num_query_retries=3): |
|
||||||
line = '-' * 80 |
|
||||||
print line |
|
||||||
print 'QPS Summary' |
|
||||||
print 'Run Id: ', self.run_id |
|
||||||
print 'Dataset Id: ', self.dataset_id |
|
||||||
print line |
|
||||||
query = ( |
|
||||||
'SELECT pod_name, recorded_at, qps FROM %s.%s WHERE run_id = \'%s\' ' |
|
||||||
'ORDER by recorded_at;') % (self.dataset_id, self.qps_table_id, |
|
||||||
self.run_id) |
|
||||||
query_job = bq_utils.sync_query_job(self.bq, self.project_id, query) |
|
||||||
print '{:<25} {:30} {}'.format('Pod name', 'Recorded at', 'Qps') |
|
||||||
print line |
|
||||||
page_token = None |
|
||||||
while True: |
|
||||||
page = self.bq.jobs().getQueryResults( |
|
||||||
pageToken=page_token, |
|
||||||
**query_job['jobReference']).execute(num_retries=num_query_retries) |
|
||||||
rows = page.get('rows', []) |
|
||||||
for row in rows: |
|
||||||
print '{:<25} {:30} {}'.format(row['f'][0]['v'], row['f'][1]['v'], |
|
||||||
row['f'][2]['v']) |
|
||||||
page_token = page.get('pageToken') |
|
||||||
if not page_token: |
|
||||||
break |
|
||||||
|
|
||||||
def __create_summary_table(self): |
|
||||||
summary_table_schema = [ |
|
||||||
('run_id', 'STRING', 'Test run id'), |
|
||||||
('image_type', 'STRING', 'Client or Server?'), |
|
||||||
('pod_name', 'STRING', 'GKE pod hosting this image'), |
|
||||||
('event_date', 'STRING', 'The date of this event'), |
|
||||||
('event_type', 'STRING', 'STARTING/RUNNING/SUCCESS/FAILURE'), |
|
||||||
('details', 'STRING', 'Any other relevant details') |
|
||||||
] |
|
||||||
desc = ('The table that contains STARTING/RUNNING/SUCCESS/FAILURE events ' |
|
||||||
'for the stress test clients and servers') |
|
||||||
return bq_utils.create_table(self.bq, self.project_id, self.dataset_id, |
|
||||||
self.summary_table_id, summary_table_schema, |
|
||||||
desc) |
|
||||||
|
|
||||||
def __create_qps_table(self): |
|
||||||
qps_table_schema = [ |
|
||||||
('run_id', 'STRING', 'Test run id'), |
|
||||||
('pod_name', 'STRING', 'GKE pod hosting this image'), |
|
||||||
('recorded_at', 'STRING', 'Metrics recorded at time'), |
|
||||||
('qps', 'INTEGER', 'Queries per second') |
|
||||||
] |
|
||||||
desc = 'The table that cointains the qps recorded at various intervals' |
|
||||||
return bq_utils.create_table(self.bq, self.project_id, self.dataset_id, |
|
||||||
self.qps_table_id, qps_table_schema, desc) |
|
||||||
@ -1,269 +0,0 @@ |
|||||||
#!/usr/bin/env python2.7 |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
import requests |
|
||||||
import json |
|
||||||
|
|
||||||
_REQUEST_TIMEOUT_SECS = 10 |
|
||||||
|
|
||||||
|
|
||||||
def _make_pod_config(pod_name, image_name, container_port_list, cmd_list, |
|
||||||
arg_list, env_dict): |
|
||||||
"""Creates a string containing the Pod defintion as required by the Kubernetes API""" |
|
||||||
body = { |
|
||||||
'kind': 'Pod', |
|
||||||
'apiVersion': 'v1', |
|
||||||
'metadata': { |
|
||||||
'name': pod_name, |
|
||||||
'labels': {'name': pod_name} |
|
||||||
}, |
|
||||||
'spec': { |
|
||||||
'containers': [ |
|
||||||
{ |
|
||||||
'name': pod_name, |
|
||||||
'image': image_name, |
|
||||||
'ports': [{'containerPort': port, |
|
||||||
'protocol': 'TCP'} |
|
||||||
for port in container_port_list], |
|
||||||
'imagePullPolicy': 'Always' |
|
||||||
} |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
env_list = [{'name': k, 'value': v} for (k, v) in env_dict.iteritems()] |
|
||||||
if len(env_list) > 0: |
|
||||||
body['spec']['containers'][0]['env'] = env_list |
|
||||||
|
|
||||||
# Add the 'Command' and 'Args' attributes if they are passed. |
|
||||||
# Note: |
|
||||||
# - 'Command' overrides the ENTRYPOINT in the Docker Image |
|
||||||
# - 'Args' override the CMD in Docker image (yes, it is confusing!) |
|
||||||
if len(cmd_list) > 0: |
|
||||||
body['spec']['containers'][0]['command'] = cmd_list |
|
||||||
if len(arg_list) > 0: |
|
||||||
body['spec']['containers'][0]['args'] = arg_list |
|
||||||
return json.dumps(body) |
|
||||||
|
|
||||||
|
|
||||||
def _make_service_config(service_name, pod_name, service_port_list, |
|
||||||
container_port_list, is_headless): |
|
||||||
"""Creates a string containing the Service definition as required by the Kubernetes API. |
|
||||||
|
|
||||||
NOTE: |
|
||||||
This creates either a Headless Service or 'LoadBalancer' service depending on |
|
||||||
the is_headless parameter. For Headless services, there is no 'type' attribute |
|
||||||
and the 'clusterIP' attribute is set to 'None'. Also, if the service is |
|
||||||
Headless, Kubernetes creates DNS entries for Pods - i.e creates DNS A-records |
|
||||||
mapping the service's name to the Pods' IPs |
|
||||||
""" |
|
||||||
if len(container_port_list) != len(service_port_list): |
|
||||||
print( |
|
||||||
'ERROR: container_port_list and service_port_list must be of same size') |
|
||||||
return '' |
|
||||||
body = { |
|
||||||
'kind': 'Service', |
|
||||||
'apiVersion': 'v1', |
|
||||||
'metadata': { |
|
||||||
'name': service_name, |
|
||||||
'labels': { |
|
||||||
'name': service_name |
|
||||||
} |
|
||||||
}, |
|
||||||
'spec': { |
|
||||||
'ports': [], |
|
||||||
'selector': { |
|
||||||
'name': pod_name |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
# Populate the 'ports' list in the 'spec' section. This maps service ports |
|
||||||
# (port numbers that are exposed by Kubernetes) to container ports (i.e port |
|
||||||
# numbers that are exposed by your Docker image) |
|
||||||
for idx in range(len(container_port_list)): |
|
||||||
port_entry = { |
|
||||||
'port': service_port_list[idx], |
|
||||||
'targetPort': container_port_list[idx], |
|
||||||
'protocol': 'TCP' |
|
||||||
} |
|
||||||
body['spec']['ports'].append(port_entry) |
|
||||||
|
|
||||||
# Make this either a LoadBalancer service or a headless service depending on |
|
||||||
# the is_headless parameter |
|
||||||
if is_headless: |
|
||||||
body['spec']['clusterIP'] = 'None' |
|
||||||
else: |
|
||||||
body['spec']['type'] = 'LoadBalancer' |
|
||||||
return json.dumps(body) |
|
||||||
|
|
||||||
|
|
||||||
def _print_connection_error(msg): |
|
||||||
print('ERROR: Connection failed. Did you remember to run Kubenetes proxy on ' |
|
||||||
'localhost (i.e kubectl proxy --port=<proxy_port>) ?. Error: %s' % msg) |
|
||||||
|
|
||||||
|
|
||||||
def _do_post(post_url, api_name, request_body): |
|
||||||
"""Helper to do HTTP POST. |
|
||||||
|
|
||||||
Note: |
|
||||||
1) On success, Kubernetes returns a success code of 201(CREATED) not 200(OK) |
|
||||||
2) A response code of 509(CONFLICT) is interpreted as a success code (since |
|
||||||
the error is most likely due to the resource already existing). This makes |
|
||||||
_do_post() idempotent which is semantically desirable. |
|
||||||
""" |
|
||||||
is_success = True |
|
||||||
try: |
|
||||||
r = requests.post(post_url, |
|
||||||
data=request_body, |
|
||||||
timeout=_REQUEST_TIMEOUT_SECS) |
|
||||||
if r.status_code == requests.codes.conflict: |
|
||||||
print('WARN: Looks like the resource already exists. Api: %s, url: %s' % |
|
||||||
(api_name, post_url)) |
|
||||||
elif r.status_code != requests.codes.created: |
|
||||||
print('ERROR: %s API returned error. HTTP response: (%d) %s' % |
|
||||||
(api_name, r.status_code, r.text)) |
|
||||||
is_success = False |
|
||||||
except (requests.exceptions.Timeout, |
|
||||||
requests.exceptions.ConnectionError) as e: |
|
||||||
is_success = False |
|
||||||
_print_connection_error(str(e)) |
|
||||||
return is_success |
|
||||||
|
|
||||||
|
|
||||||
def _do_delete(del_url, api_name): |
|
||||||
"""Helper to do HTTP DELETE. |
|
||||||
|
|
||||||
Note: A response code of 404(NOT_FOUND) is treated as success to keep |
|
||||||
_do_delete() idempotent. |
|
||||||
""" |
|
||||||
is_success = True |
|
||||||
try: |
|
||||||
r = requests.delete(del_url, timeout=_REQUEST_TIMEOUT_SECS) |
|
||||||
if r.status_code == requests.codes.not_found: |
|
||||||
print('WARN: The resource does not exist. Api: %s, url: %s' % |
|
||||||
(api_name, del_url)) |
|
||||||
elif r.status_code != requests.codes.ok: |
|
||||||
print('ERROR: %s API returned error. HTTP response: %s' % |
|
||||||
(api_name, r.text)) |
|
||||||
is_success = False |
|
||||||
except (requests.exceptions.Timeout, |
|
||||||
requests.exceptions.ConnectionError) as e: |
|
||||||
is_success = False |
|
||||||
_print_connection_error(str(e)) |
|
||||||
return is_success |
|
||||||
|
|
||||||
|
|
||||||
def create_service(kube_host, kube_port, namespace, service_name, pod_name, |
|
||||||
service_port_list, container_port_list, is_headless): |
|
||||||
"""Creates either a Headless Service or a LoadBalancer Service depending |
|
||||||
on the is_headless parameter. |
|
||||||
""" |
|
||||||
post_url = 'http://%s:%d/api/v1/namespaces/%s/services' % ( |
|
||||||
kube_host, kube_port, namespace) |
|
||||||
request_body = _make_service_config(service_name, pod_name, service_port_list, |
|
||||||
container_port_list, is_headless) |
|
||||||
return _do_post(post_url, 'Create Service', request_body) |
|
||||||
|
|
||||||
|
|
||||||
def create_pod(kube_host, kube_port, namespace, pod_name, image_name, |
|
||||||
container_port_list, cmd_list, arg_list, env_dict): |
|
||||||
"""Creates a Kubernetes Pod. |
|
||||||
|
|
||||||
Note that it is generally NOT considered a good practice to directly create |
|
||||||
Pods. Typically, the recommendation is to create 'Controllers' to create and |
|
||||||
manage Pods' lifecycle. Currently Kubernetes only supports 'Replication |
|
||||||
Controller' which creates a configurable number of 'identical Replicas' of |
|
||||||
Pods and automatically restarts any Pods in case of failures (for eg: Machine |
|
||||||
failures in Kubernetes). This makes it less flexible for our test use cases |
|
||||||
where we might want slightly different set of args to each Pod. Hence we |
|
||||||
directly create Pods and not care much about Kubernetes failures since those |
|
||||||
are very rare. |
|
||||||
""" |
|
||||||
post_url = 'http://%s:%d/api/v1/namespaces/%s/pods' % (kube_host, kube_port, |
|
||||||
namespace) |
|
||||||
request_body = _make_pod_config(pod_name, image_name, container_port_list, |
|
||||||
cmd_list, arg_list, env_dict) |
|
||||||
return _do_post(post_url, 'Create Pod', request_body) |
|
||||||
|
|
||||||
|
|
||||||
def delete_service(kube_host, kube_port, namespace, service_name): |
|
||||||
del_url = 'http://%s:%d/api/v1/namespaces/%s/services/%s' % ( |
|
||||||
kube_host, kube_port, namespace, service_name) |
|
||||||
return _do_delete(del_url, 'Delete Service') |
|
||||||
|
|
||||||
|
|
||||||
def delete_pod(kube_host, kube_port, namespace, pod_name): |
|
||||||
del_url = 'http://%s:%d/api/v1/namespaces/%s/pods/%s' % (kube_host, kube_port, |
|
||||||
namespace, pod_name) |
|
||||||
return _do_delete(del_url, 'Delete Pod') |
|
||||||
|
|
||||||
|
|
||||||
def create_pod_and_service(kube_host, kube_port, namespace, pod_name, |
|
||||||
image_name, container_port_list, cmd_list, arg_list, |
|
||||||
env_dict, is_headless_service): |
|
||||||
"""A helper function that creates a pod and a service (if pod creation was successful).""" |
|
||||||
is_success = create_pod(kube_host, kube_port, namespace, pod_name, image_name, |
|
||||||
container_port_list, cmd_list, arg_list, env_dict) |
|
||||||
if not is_success: |
|
||||||
print 'Error in creating Pod' |
|
||||||
return False |
|
||||||
|
|
||||||
is_success = create_service( |
|
||||||
kube_host, |
|
||||||
kube_port, |
|
||||||
namespace, |
|
||||||
pod_name, # Use pod_name for service |
|
||||||
pod_name, |
|
||||||
container_port_list, # Service port list same as container port list |
|
||||||
container_port_list, |
|
||||||
is_headless_service) |
|
||||||
if not is_success: |
|
||||||
print 'Error in creating Service' |
|
||||||
return False |
|
||||||
|
|
||||||
print 'Successfully created the pod/service %s' % pod_name |
|
||||||
return True |
|
||||||
|
|
||||||
|
|
||||||
def delete_pod_and_service(kube_host, kube_port, namespace, pod_name): |
|
||||||
""" A helper function that calls delete_pod and delete_service """ |
|
||||||
is_success = delete_pod(kube_host, kube_port, namespace, pod_name) |
|
||||||
if not is_success: |
|
||||||
print 'Error in deleting pod %s' % pod_name |
|
||||||
return False |
|
||||||
|
|
||||||
# Note: service name assumed to the the same as pod name |
|
||||||
is_success = delete_service(kube_host, kube_port, namespace, pod_name) |
|
||||||
if not is_success: |
|
||||||
print 'Error in deleting service %s' % pod_name |
|
||||||
return False |
|
||||||
|
|
||||||
print 'Successfully deleted the Pod/Service: %s' % pod_name |
|
||||||
return True |
|
||||||
@ -1,37 +0,0 @@ |
|||||||
#!/usr/bin/env bash |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
# |
|
||||||
# This script is invoked by Jenkins and runs interop test suite. |
|
||||||
set -ex |
|
||||||
|
|
||||||
# Enter the gRPC repo root |
|
||||||
cd $(dirname $0)/../.. |
|
||||||
|
|
||||||
tools/run_tests/run_stress_tests.py -l all -s all -j 12 $@ || true |
|
||||||
@ -1,331 +0,0 @@ |
|||||||
#!/usr/bin/env python |
|
||||||
# Copyright 2015, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
"""Run stress test in C++""" |
|
||||||
|
|
||||||
from __future__ import print_function |
|
||||||
|
|
||||||
import argparse |
|
||||||
import atexit |
|
||||||
import itertools |
|
||||||
import json |
|
||||||
import multiprocessing |
|
||||||
import os |
|
||||||
import re |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
import tempfile |
|
||||||
import time |
|
||||||
import uuid |
|
||||||
import six |
|
||||||
|
|
||||||
import python_utils.dockerjob as dockerjob |
|
||||||
import python_utils.jobset as jobset |
|
||||||
|
|
||||||
# Docker doesn't clean up after itself, so we do it on exit. |
|
||||||
atexit.register(lambda: subprocess.call(['stty', 'echo'])) |
|
||||||
|
|
||||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) |
|
||||||
os.chdir(ROOT) |
|
||||||
|
|
||||||
_DEFAULT_SERVER_PORT = 8080 |
|
||||||
_DEFAULT_METRICS_PORT = 8081 |
|
||||||
_DEFAULT_TEST_CASES = 'empty_unary:20,large_unary:20,client_streaming:20,server_streaming:20,empty_stream:20' |
|
||||||
_DEFAULT_NUM_CHANNELS_PER_SERVER = 5 |
|
||||||
_DEFAULT_NUM_STUBS_PER_CHANNEL = 10 |
|
||||||
|
|
||||||
# 15 mins default |
|
||||||
_DEFAULT_TEST_DURATION_SECS = 900 |
|
||||||
|
|
||||||
class CXXLanguage: |
|
||||||
|
|
||||||
def __init__(self): |
|
||||||
self.client_cwd = None |
|
||||||
self.server_cwd = None |
|
||||||
self.safename = 'cxx' |
|
||||||
|
|
||||||
def client_cmd(self, args): |
|
||||||
return ['bins/opt/stress_test'] + args |
|
||||||
|
|
||||||
def server_cmd(self, args): |
|
||||||
return ['bins/opt/interop_server'] + args |
|
||||||
|
|
||||||
def global_env(self): |
|
||||||
return {} |
|
||||||
|
|
||||||
def __str__(self): |
|
||||||
return 'c++' |
|
||||||
|
|
||||||
|
|
||||||
_LANGUAGES = {'c++': CXXLanguage(),} |
|
||||||
|
|
||||||
# languages supported as cloud_to_cloud servers |
|
||||||
_SERVERS = ['c++'] |
|
||||||
|
|
||||||
DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' |
|
||||||
|
|
||||||
|
|
||||||
def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): |
|
||||||
"""Wraps given cmdline array to create 'docker run' cmdline from it.""" |
|
||||||
docker_cmdline = ['docker', 'run', '-i', '--rm=true'] |
|
||||||
|
|
||||||
# turn environ into -e docker args |
|
||||||
if environ: |
|
||||||
for k, v in environ.items(): |
|
||||||
docker_cmdline += ['-e', '%s=%s' % (k, v)] |
|
||||||
|
|
||||||
# set working directory |
|
||||||
workdir = DOCKER_WORKDIR_ROOT |
|
||||||
if cwd: |
|
||||||
workdir = os.path.join(workdir, cwd) |
|
||||||
docker_cmdline += ['-w', workdir] |
|
||||||
|
|
||||||
docker_cmdline += docker_args + [image] + cmdline |
|
||||||
return docker_cmdline |
|
||||||
|
|
||||||
|
|
||||||
def bash_login_cmdline(cmdline): |
|
||||||
"""Creates bash -l -c cmdline from args list.""" |
|
||||||
# Use login shell: |
|
||||||
# * rvm and nvm require it |
|
||||||
# * makes error messages clearer if executables are missing |
|
||||||
return ['bash', '-l', '-c', ' '.join(cmdline)] |
|
||||||
|
|
||||||
|
|
||||||
def _job_kill_handler(job): |
|
||||||
if job._spec.container_name: |
|
||||||
dockerjob.docker_kill(job._spec.container_name) |
|
||||||
# When the job times out and we decide to kill it, |
|
||||||
# we need to wait a before restarting the job |
|
||||||
# to prevent "container name already in use" error. |
|
||||||
# TODO(jtattermusch): figure out a cleaner way to to this. |
|
||||||
time.sleep(2) |
|
||||||
|
|
||||||
|
|
||||||
def cloud_to_cloud_jobspec(language, |
|
||||||
test_cases, |
|
||||||
server_addresses, |
|
||||||
test_duration_secs, |
|
||||||
num_channels_per_server, |
|
||||||
num_stubs_per_channel, |
|
||||||
metrics_port, |
|
||||||
docker_image=None): |
|
||||||
"""Creates jobspec for cloud-to-cloud interop test""" |
|
||||||
cmdline = bash_login_cmdline(language.client_cmd([ |
|
||||||
'--test_cases=%s' % test_cases, '--server_addresses=%s' % |
|
||||||
server_addresses, '--test_duration_secs=%s' % test_duration_secs, |
|
||||||
'--num_stubs_per_channel=%s' % num_stubs_per_channel, |
|
||||||
'--num_channels_per_server=%s' % num_channels_per_server, |
|
||||||
'--metrics_port=%s' % metrics_port |
|
||||||
])) |
|
||||||
print(cmdline) |
|
||||||
cwd = language.client_cwd |
|
||||||
environ = language.global_env() |
|
||||||
if docker_image: |
|
||||||
container_name = dockerjob.random_name('interop_client_%s' % |
|
||||||
language.safename) |
|
||||||
cmdline = docker_run_cmdline( |
|
||||||
cmdline, |
|
||||||
image=docker_image, |
|
||||||
environ=environ, |
|
||||||
cwd=cwd, |
|
||||||
docker_args=['--net=host', '--name', container_name]) |
|
||||||
cwd = None |
|
||||||
|
|
||||||
test_job = jobset.JobSpec(cmdline=cmdline, |
|
||||||
cwd=cwd, |
|
||||||
environ=environ, |
|
||||||
shortname='cloud_to_cloud:%s:%s_server:stress_test' % ( |
|
||||||
language, server_name), |
|
||||||
timeout_seconds=test_duration_secs * 2, |
|
||||||
flake_retries=0, |
|
||||||
timeout_retries=0, |
|
||||||
kill_handler=_job_kill_handler) |
|
||||||
test_job.container_name = container_name |
|
||||||
return test_job |
|
||||||
|
|
||||||
|
|
||||||
def server_jobspec(language, docker_image, test_duration_secs): |
|
||||||
"""Create jobspec for running a server""" |
|
||||||
container_name = dockerjob.random_name('interop_server_%s' % |
|
||||||
language.safename) |
|
||||||
cmdline = bash_login_cmdline(language.server_cmd(['--port=%s' % |
|
||||||
_DEFAULT_SERVER_PORT])) |
|
||||||
environ = language.global_env() |
|
||||||
docker_cmdline = docker_run_cmdline( |
|
||||||
cmdline, |
|
||||||
image=docker_image, |
|
||||||
cwd=language.server_cwd, |
|
||||||
environ=environ, |
|
||||||
docker_args=['-p', str(_DEFAULT_SERVER_PORT), '--name', container_name]) |
|
||||||
|
|
||||||
server_job = jobset.JobSpec(cmdline=docker_cmdline, |
|
||||||
environ=environ, |
|
||||||
shortname='interop_server_%s' % language, |
|
||||||
timeout_seconds=test_duration_secs * 3) |
|
||||||
server_job.container_name = container_name |
|
||||||
return server_job |
|
||||||
|
|
||||||
|
|
||||||
def build_interop_stress_image_jobspec(language, tag=None): |
|
||||||
"""Creates jobspec for building stress test docker image for a language""" |
|
||||||
if not tag: |
|
||||||
tag = 'grpc_interop_stress_%s:%s' % (language.safename, uuid.uuid4()) |
|
||||||
env = {'INTEROP_IMAGE': tag, |
|
||||||
'BASE_NAME': 'grpc_interop_stress_%s' % language.safename} |
|
||||||
build_job = jobset.JobSpec(cmdline=['tools/run_tests/dockerize/build_interop_stress_image.sh'], |
|
||||||
environ=env, |
|
||||||
shortname='build_docker_%s' % (language), |
|
||||||
timeout_seconds=30 * 60) |
|
||||||
build_job.tag = tag |
|
||||||
return build_job |
|
||||||
|
|
||||||
argp = argparse.ArgumentParser(description='Run stress tests.') |
|
||||||
argp.add_argument('-l', |
|
||||||
'--language', |
|
||||||
choices=['all'] + sorted(_LANGUAGES), |
|
||||||
nargs='+', |
|
||||||
default=['all'], |
|
||||||
help='Clients to run.') |
|
||||||
argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) |
|
||||||
argp.add_argument( |
|
||||||
'-s', |
|
||||||
'--server', |
|
||||||
choices=['all'] + sorted(_SERVERS), |
|
||||||
action='append', |
|
||||||
help='Run cloud_to_cloud servers in a separate docker ' + 'image.', |
|
||||||
default=[]) |
|
||||||
argp.add_argument( |
|
||||||
'--override_server', |
|
||||||
action='append', |
|
||||||
type=lambda kv: kv.split('='), |
|
||||||
help= |
|
||||||
'Use servername=HOST:PORT to explicitly specify a server. E.g. ' |
|
||||||
'csharp=localhost:50000', |
|
||||||
default=[]) |
|
||||||
argp.add_argument('--test_duration_secs', |
|
||||||
help='The duration of the test in seconds', |
|
||||||
default=_DEFAULT_TEST_DURATION_SECS) |
|
||||||
|
|
||||||
args = argp.parse_args() |
|
||||||
|
|
||||||
servers = set( |
|
||||||
s |
|
||||||
for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x] |
|
||||||
for x in args.server)) |
|
||||||
|
|
||||||
languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( |
|
||||||
six.iterkeys(_LANGUAGES) if x == 'all' else [x] for x in args.language)) |
|
||||||
|
|
||||||
docker_images = {} |
|
||||||
# languages for which to build docker images |
|
||||||
languages_to_build = set( |
|
||||||
_LANGUAGES[k] |
|
||||||
for k in set([str(l) for l in languages] + [s for s in servers])) |
|
||||||
build_jobs = [] |
|
||||||
for l in languages_to_build: |
|
||||||
job = build_interop_stress_image_jobspec(l) |
|
||||||
docker_images[str(l)] = job.tag |
|
||||||
build_jobs.append(job) |
|
||||||
|
|
||||||
if build_jobs: |
|
||||||
jobset.message('START', 'Building interop docker images.', do_newline=True) |
|
||||||
num_failures, _ = jobset.run(build_jobs, |
|
||||||
newline_on_success=True, |
|
||||||
maxjobs=args.jobs) |
|
||||||
if num_failures == 0: |
|
||||||
jobset.message('SUCCESS', |
|
||||||
'All docker images built successfully.', |
|
||||||
do_newline=True) |
|
||||||
else: |
|
||||||
jobset.message('FAILED', |
|
||||||
'Failed to build interop docker images.', |
|
||||||
do_newline=True) |
|
||||||
for image in six.itervalues(docker_images): |
|
||||||
dockerjob.remove_image(image, skip_nonexistent=True) |
|
||||||
sys.exit(1) |
|
||||||
|
|
||||||
# Start interop servers. |
|
||||||
server_jobs = {} |
|
||||||
server_addresses = {} |
|
||||||
try: |
|
||||||
for s in servers: |
|
||||||
lang = str(s) |
|
||||||
spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang), args.test_duration_secs) |
|
||||||
job = dockerjob.DockerJob(spec) |
|
||||||
server_jobs[lang] = job |
|
||||||
server_addresses[lang] = ('localhost', |
|
||||||
job.mapped_port(_DEFAULT_SERVER_PORT)) |
|
||||||
|
|
||||||
jobs = [] |
|
||||||
|
|
||||||
for server in args.override_server: |
|
||||||
server_name = server[0] |
|
||||||
(server_host, server_port) = server[1].split(':') |
|
||||||
server_addresses[server_name] = (server_host, server_port) |
|
||||||
|
|
||||||
for server_name, server_address in server_addresses.items(): |
|
||||||
(server_host, server_port) = server_address |
|
||||||
for language in languages: |
|
||||||
test_job = cloud_to_cloud_jobspec( |
|
||||||
language, |
|
||||||
_DEFAULT_TEST_CASES, |
|
||||||
('%s:%s' % (server_host, server_port)), |
|
||||||
args.test_duration_secs, |
|
||||||
_DEFAULT_NUM_CHANNELS_PER_SERVER, |
|
||||||
_DEFAULT_NUM_STUBS_PER_CHANNEL, |
|
||||||
_DEFAULT_METRICS_PORT, |
|
||||||
docker_image=docker_images.get(str(language))) |
|
||||||
jobs.append(test_job) |
|
||||||
|
|
||||||
if not jobs: |
|
||||||
print('No jobs to run.') |
|
||||||
for image in six.itervalues(docker_images): |
|
||||||
dockerjob.remove_image(image, skip_nonexistent=True) |
|
||||||
sys.exit(1) |
|
||||||
|
|
||||||
num_failures, resultset = jobset.run(jobs, |
|
||||||
newline_on_success=True, |
|
||||||
maxjobs=args.jobs) |
|
||||||
if num_failures: |
|
||||||
jobset.message('FAILED', 'Some tests failed', do_newline=True) |
|
||||||
else: |
|
||||||
jobset.message('SUCCESS', 'All tests passed', do_newline=True) |
|
||||||
|
|
||||||
finally: |
|
||||||
# Check if servers are still running. |
|
||||||
for server, job in server_jobs.items(): |
|
||||||
if not job.is_running(): |
|
||||||
print('Server "%s" has exited prematurely.' % server) |
|
||||||
|
|
||||||
dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) |
|
||||||
|
|
||||||
for image in six.itervalues(docker_images): |
|
||||||
print('Removing docker image %s' % image) |
|
||||||
dockerjob.remove_image(image) |
|
||||||
@ -1,85 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_cxx_asan" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_cxx", |
|
||||||
"buildType": "asan" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 120, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_client_asan": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": ["/var/local/git/grpc/bins/asan/stress_test"], |
|
||||||
"metricsClientCmd": ["/var/local/git/grpc/bins/asan/metrics_client"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_server_asan": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": ["/var/local/git/grpc/bins/asan/interop_server"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"stress-server-asan": { |
|
||||||
"serverTemplate": "cxx_server_asan", |
|
||||||
"dockerImage": "grpc_stress_cxx_asan", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"stress-client-asan": { |
|
||||||
"clientTemplate": "cxx_client_asan", |
|
||||||
"dockerImage": "grpc_stress_cxx_asan", |
|
||||||
"numInstances": 5, |
|
||||||
"serverPodSpec": "stress-server-asan" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8003, |
|
||||||
"datasetIdNamePrefix": "stress_test_asan", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,91 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_csharp" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_csharp" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 100, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true", |
|
||||||
"deadline_secs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"csharp_client": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": [ |
|
||||||
"mono", |
|
||||||
"/var/local/git/grpc/src/csharp/Grpc.IntegrationTesting.StressClient/bin/Debug/Grpc.IntegrationTesting.StressClient.exe" |
|
||||||
], |
|
||||||
"metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"csharp_server": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": [ |
|
||||||
"mono", |
|
||||||
"/var/local/git/grpc/src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/Grpc.IntegrationTesting.Server.exe" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"stress-server-csharp": { |
|
||||||
"serverTemplate": "csharp_server", |
|
||||||
"dockerImage": "grpc_stress_csharp", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"stress-client-csharp": { |
|
||||||
"clientTemplate": "csharp_client", |
|
||||||
"dockerImage": "grpc_stress_csharp", |
|
||||||
"numInstances": 10, |
|
||||||
"serverPodSpec": "stress-server-csharp" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 100, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8009, |
|
||||||
"datasetIdNamePrefix": "stress_test_csharp", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,96 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_go" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_go" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"go_client": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": [ |
|
||||||
"go", |
|
||||||
"run", |
|
||||||
"/go/src/google.golang.org/grpc/stress/client/main.go" |
|
||||||
], |
|
||||||
"metricsClientCmd": [ |
|
||||||
"go", |
|
||||||
"run", |
|
||||||
"/go/src/google.golang.org/grpc/stress/metrics_client/main.go" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"go_server": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": [ |
|
||||||
"go", |
|
||||||
"run", |
|
||||||
"/go/src/google.golang.org/grpc/interop/server/server.go" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"go-stress-server": { |
|
||||||
"serverTemplate": "go_server", |
|
||||||
"dockerImage": "grpc_stress_go", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"go-stress-client": { |
|
||||||
"clientTemplate": "go_client", |
|
||||||
"dockerImage": "grpc_stress_go", |
|
||||||
"numInstances": 15, |
|
||||||
"serverPodSpec": "go-stress-server" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8007, |
|
||||||
"datasetIdNamePrefix": "stress_test_go", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,98 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_java" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_java" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 100, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true", |
|
||||||
"deadline_secs": 60 |
|
||||||
}, |
|
||||||
"env": { |
|
||||||
"STRESSTEST_CLIENT_OPTS":"-Xmx3g -Xms3g -XX:NewSize=1500m -XX:MaxNewSize=1500m -XX:+UseConcMarkSweepGC" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"java_client": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": [ |
|
||||||
"/var/local/git/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/stresstest-client" |
|
||||||
], |
|
||||||
"metricsClientCmd": [ |
|
||||||
"/var/local/git/grpc/bins/opt/metrics_client" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080, |
|
||||||
"use_tls": "false" |
|
||||||
}, |
|
||||||
"env": { |
|
||||||
"TEST_SERVER_OPTS":"-Xmx3g -Xms3g -XX:NewSize=1500m -XX:MaxNewSize=1500m -XX:+UseConcMarkSweepGC" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"java_server": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": [ |
|
||||||
"/var/local/git/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/test-server" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"java-stress-server": { |
|
||||||
"serverTemplate": "java_server", |
|
||||||
"dockerImage": "grpc_stress_java", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"java-stress-client": { |
|
||||||
"clientTemplate": "java_client", |
|
||||||
"dockerImage": "grpc_stress_java", |
|
||||||
"numInstances": 10, |
|
||||||
"serverPodSpec": "java-stress-server" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 100, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8008, |
|
||||||
"datasetIdNamePrefix": "stress_test_java", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,97 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_cxx_opt" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_cxx", |
|
||||||
"buildType": "opt" |
|
||||||
}, |
|
||||||
"grpc_stress_node": { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_node" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"node_client": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": [ |
|
||||||
"/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", |
|
||||||
"node", |
|
||||||
"/var/local/git/grpc/src/node/stress/stress_client.js" |
|
||||||
], |
|
||||||
"metricsClientCmd": [ |
|
||||||
"/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", |
|
||||||
"node", |
|
||||||
"/var/local/git/grpc/src/node/stress/metrics_client.js" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_server_opt": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"stress-server-cxx-opt": { |
|
||||||
"serverTemplate": "cxx_server_opt", |
|
||||||
"dockerImage": "grpc_stress_cxx_opt", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"stress-client-node": { |
|
||||||
"clientTemplate": "node_client", |
|
||||||
"dockerImage": "grpc_stress_node", |
|
||||||
"numInstances": 20, |
|
||||||
"serverPodSpec": "stress-server-cxx-opt" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8006, |
|
||||||
"datasetIdNamePrefix": "stress_test_node_cxx_opt", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,96 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_node" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_node" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"node_client": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": [ |
|
||||||
"/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", |
|
||||||
"node", |
|
||||||
"/var/local/git/grpc/src/node/stress/stress_client.js" |
|
||||||
], |
|
||||||
"metricsClientCmd": [ |
|
||||||
"/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", |
|
||||||
"node", |
|
||||||
"/var/local/git/grpc/src/node/stress/metrics_client.js" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"node_server": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": [ |
|
||||||
"/var/local/git/grpc/tools/gcp/stress_test/run_node.sh", |
|
||||||
"node", |
|
||||||
"/var/local/git/grpc/src/node/interop/interop_server.js" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"node-stress-server": { |
|
||||||
"serverTemplate": "node_server", |
|
||||||
"dockerImage": "grpc_stress_node", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"node-stress-client": { |
|
||||||
"clientTemplate": "node_client", |
|
||||||
"dockerImage": "grpc_stress_node", |
|
||||||
"numInstances": 15, |
|
||||||
"serverPodSpec": "node-stress-server" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8005, |
|
||||||
"datasetIdNamePrefix": "stress_test_node", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,134 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_cxx_opt" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_cxx", |
|
||||||
"buildType": "opt" |
|
||||||
}, |
|
||||||
"grpc_stress_cxx_tsan": { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_cxx", |
|
||||||
"buildType": "tsan" |
|
||||||
}, |
|
||||||
"grpc_stress_cxx_asan": { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_cxx", |
|
||||||
"buildType": "asan" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_client_opt": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": ["/var/local/git/grpc/bins/opt/stress_test"], |
|
||||||
"metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] |
|
||||||
}, |
|
||||||
"cxx_client_tsan": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": ["/var/local/git/grpc/bins/tsan/stress_test"], |
|
||||||
"metricsClientCmd": ["/var/local/git/grpc/bins/tsan/metrics_client"] |
|
||||||
}, |
|
||||||
"cxx_client_asan": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": ["/var/local/git/grpc/bins/asan/stress_test"], |
|
||||||
"metricsClientCmd": ["/var/local/git/grpc/bins/asan/metrics_client"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_server_opt": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] |
|
||||||
}, |
|
||||||
"cxx_server_tsan": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": ["/var/local/git/grpc/bins/tsan/interop_server"] |
|
||||||
}, |
|
||||||
"cxx_server_asan": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": ["/var/local/git/grpc/bins/asan/interop_server"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"stress-server-opt": { |
|
||||||
"serverTemplate": "cxx_server_opt", |
|
||||||
"dockerImage": "grpc_stress_cxx_opt", |
|
||||||
"numInstances": 1 |
|
||||||
}, |
|
||||||
"stress-server-tsan": { |
|
||||||
"serverTemplate": "cxx_server_tsan", |
|
||||||
"dockerImage": "grpc_stress_cxx_tsan", |
|
||||||
"numInstances": 1 |
|
||||||
}, |
|
||||||
"stress-server-asan": { |
|
||||||
"serverTemplate": "cxx_server_asan", |
|
||||||
"dockerImage": "grpc_stress_cxx_asan", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"stress-client-opt": { |
|
||||||
"clientTemplate": "cxx_client_opt", |
|
||||||
"dockerImage": "grpc_stress_cxx_opt", |
|
||||||
"numInstances": 5, |
|
||||||
"serverPodSpec": "stress-server-opt" |
|
||||||
}, |
|
||||||
"stress-client-tsan": { |
|
||||||
"clientTemplate": "cxx_client_tsan", |
|
||||||
"dockerImage": "grpc_stress_cxx_tsan", |
|
||||||
"numInstances": 10, |
|
||||||
"serverPodSpec": "stress-server-tsan" |
|
||||||
}, |
|
||||||
"stress-client-asan": { |
|
||||||
"clientTemplate": "cxx_client_asan", |
|
||||||
"dockerImage": "grpc_stress_cxx_asan", |
|
||||||
"numInstances": 10, |
|
||||||
"serverPodSpec": "stress-server-asan" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8004, |
|
||||||
"datasetIdNamePrefix": "stress_test_opt_tsan", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,85 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_cxx_opt" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_cxx", |
|
||||||
"buildType": "opt" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_client_opt": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": ["/var/local/git/grpc/bins/opt/stress_test"], |
|
||||||
"metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_server_opt": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"stress-server-opt": { |
|
||||||
"serverTemplate": "cxx_server_opt", |
|
||||||
"dockerImage": "grpc_stress_cxx_opt", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"stress-client-opt": { |
|
||||||
"clientTemplate": "cxx_client_opt", |
|
||||||
"dockerImage": "grpc_stress_cxx_opt", |
|
||||||
"numInstances": 15, |
|
||||||
"serverPodSpec": "stress-server-opt" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8001, |
|
||||||
"datasetIdNamePrefix": "stress_test_opt", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,93 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_cxx_opt" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_cxx", |
|
||||||
"buildType": "opt" |
|
||||||
}, |
|
||||||
"grpc_stress_php": { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_php" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"php_client": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": [ |
|
||||||
"/var/local/git/grpc/src/php/bin/stress_client.sh" |
|
||||||
], |
|
||||||
"metricsClientCmd": [ |
|
||||||
"php", |
|
||||||
"/var/local/git/grpc/src/php/tests/interop/metrics_client.php" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_server_opt": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"stress-server-cxx-php": { |
|
||||||
"serverTemplate": "cxx_server_opt", |
|
||||||
"dockerImage": "grpc_stress_cxx_opt", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"stress-client-php": { |
|
||||||
"clientTemplate": "php_client", |
|
||||||
"dockerImage": "grpc_stress_php", |
|
||||||
"numInstances": 20, |
|
||||||
"serverPodSpec": "stress-server-cxx-php" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8010, |
|
||||||
"datasetIdNamePrefix": "stress_test_php_cxx_opt", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,98 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_python" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_python" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true" |
|
||||||
}, |
|
||||||
"env": { |
|
||||||
"PYTHONPATH": "/var/local/git/grpc/src/python/gens:/var/local/git/grpc/src/python/grpcio", |
|
||||||
"LD_LIBRARY_PATH":"/var/local/git/grpc/libs/opt" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"python_client": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": [ |
|
||||||
"python", |
|
||||||
"/var/local/git/grpc/src/python/grpcio/tests/stress/client.py" |
|
||||||
], |
|
||||||
"metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
}, |
|
||||||
"env": { |
|
||||||
"PYTHONPATH": "/var/local/git/grpc/src/python/gens:/var/local/git/grpc/src/python/grpcio", |
|
||||||
"LD_LIBRARY_PATH":"/var/local/git/grpc/libs/opt" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"python_server": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": [ |
|
||||||
"python", |
|
||||||
"/var/local/git/grpc/src/python/grpcio/tests/interop/server.py" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"python-stress-server": { |
|
||||||
"serverTemplate": "python_server", |
|
||||||
"dockerImage": "grpc_stress_python", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"python-stress-client": { |
|
||||||
"clientTemplate": "python_client", |
|
||||||
"dockerImage": "grpc_stress_python", |
|
||||||
"numInstances": 5, |
|
||||||
"serverPodSpec": "python-stress-server" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8011, |
|
||||||
"datasetIdNamePrefix": "stress_test_python", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,92 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_ruby" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_ruby" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"ruby_client": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": [ |
|
||||||
"/var/local/git/grpc/tools/gcp/stress_test/run_ruby.sh", |
|
||||||
"ruby", |
|
||||||
"/var/local/git/grpc/src/ruby/stress/stress_client.rb" |
|
||||||
], |
|
||||||
"metricsClientCmd": ["/var/local/git/grpc/bins/opt/metrics_client"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"ruby_server": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": [ |
|
||||||
"/var/local/git/grpc/tools/gcp/stress_test/run_ruby.sh", |
|
||||||
"ruby", |
|
||||||
"/var/local/git/grpc/src/ruby/pb/test/server.rb" |
|
||||||
] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"stress-server-ruby": { |
|
||||||
"serverTemplate": "ruby_server", |
|
||||||
"dockerImage": "grpc_stress_ruby", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"stress-client-ruby": { |
|
||||||
"clientTemplate": "ruby_client", |
|
||||||
"dockerImage": "grpc_stress_ruby", |
|
||||||
"numInstances": 10, |
|
||||||
"serverPodSpec": "stress-server-ruby" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8001, |
|
||||||
"datasetIdNamePrefix": "stress_test_ruby", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,85 +0,0 @@ |
|||||||
{ |
|
||||||
"dockerImages": { |
|
||||||
"grpc_stress_cxx_tsan" : { |
|
||||||
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh", |
|
||||||
"dockerFileDir": "grpc_interop_stress_cxx", |
|
||||||
"buildType": "tsan" |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientTemplates": { |
|
||||||
"baseTemplates": { |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", |
|
||||||
"pollIntervalSecs": 120, |
|
||||||
"clientArgs": { |
|
||||||
"num_channels_per_server":5, |
|
||||||
"num_stubs_per_channel":10, |
|
||||||
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", |
|
||||||
"metrics_port": 8081 |
|
||||||
}, |
|
||||||
"metricsPort": 8081, |
|
||||||
"metricsArgs": { |
|
||||||
"metrics_server_address": "localhost:8081", |
|
||||||
"total_only": "true" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_client_tsan": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressClientCmd": ["/var/local/git/grpc/bins/tsan/stress_test"], |
|
||||||
"metricsClientCmd": ["/var/local/git/grpc/bins/tsan/metrics_client"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"serverTemplates": { |
|
||||||
"baseTemplates":{ |
|
||||||
"default": { |
|
||||||
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py", |
|
||||||
"serverPort": 8080, |
|
||||||
"serverArgs": { |
|
||||||
"port": 8080 |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
"templates": { |
|
||||||
"cxx_server_tsan": { |
|
||||||
"baseTemplate": "default", |
|
||||||
"stressServerCmd": ["/var/local/git/grpc/bins/tsan/interop_server"] |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"testMatrix": { |
|
||||||
"serverPodSpecs": { |
|
||||||
"stress-server-tsan": { |
|
||||||
"serverTemplate": "cxx_server_tsan", |
|
||||||
"dockerImage": "grpc_stress_cxx_tsan", |
|
||||||
"numInstances": 1 |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"clientPodSpecs": { |
|
||||||
"stress-client-tsan": { |
|
||||||
"clientTemplate": "cxx_client_tsan", |
|
||||||
"dockerImage": "grpc_stress_cxx_tsan", |
|
||||||
"numInstances": 5, |
|
||||||
"serverPodSpec": "stress-server-tsan" |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
|
|
||||||
"globalSettings": { |
|
||||||
"buildDockerImages": true, |
|
||||||
"pollIntervalSecs": 60, |
|
||||||
"testDurationSecs": 7200, |
|
||||||
"kubernetesProxyPort": 8002, |
|
||||||
"datasetIdNamePrefix": "stress_test_tsan", |
|
||||||
"summaryTableId": "summary", |
|
||||||
"qpsTableId": "qps", |
|
||||||
"podWarmupSecs": 60 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
@ -1,59 +0,0 @@ |
|||||||
#!/usr/bin/env python |
|
||||||
# Copyright 2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
import argparse |
|
||||||
import os |
|
||||||
import sys |
|
||||||
|
|
||||||
stress_test_utils_dir = os.path.abspath(os.path.join( |
|
||||||
os.path.dirname(__file__), '../../gcp/stress_test')) |
|
||||||
sys.path.append(stress_test_utils_dir) |
|
||||||
from stress_test_utils import BigQueryHelper |
|
||||||
|
|
||||||
argp = argparse.ArgumentParser( |
|
||||||
description='Print summary tables', |
|
||||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
|
||||||
argp.add_argument('--gcp_project_id', |
|
||||||
required=True, |
|
||||||
help='The Google Cloud Platform Project Id') |
|
||||||
argp.add_argument('--dataset_id', type=str, required=True) |
|
||||||
argp.add_argument('--run_id', type=str, required=True) |
|
||||||
argp.add_argument('--summary_table_id', type=str, default='summary') |
|
||||||
argp.add_argument('--qps_table_id', type=str, default='qps') |
|
||||||
argp.add_argument('--summary_only', action='store_true', default=True) |
|
||||||
|
|
||||||
if __name__ == '__main__': |
|
||||||
args = argp.parse_args() |
|
||||||
bq_helper = BigQueryHelper(args.run_id, '', '', args.gcp_project_id, |
|
||||||
args.dataset_id, args.summary_table_id, |
|
||||||
args.qps_table_id) |
|
||||||
bq_helper.initialize() |
|
||||||
if not args.summary_only: |
|
||||||
bq_helper.print_qps_records() |
|
||||||
bq_helper.print_summary_records() |
|
||||||
@ -1,674 +0,0 @@ |
|||||||
#!/usr/bin/env python |
|
||||||
# Copyright 2015-2016, Google Inc. |
|
||||||
# All rights reserved. |
|
||||||
# |
|
||||||
# Redistribution and use in source and binary forms, with or without |
|
||||||
# modification, are permitted provided that the following conditions are |
|
||||||
# met: |
|
||||||
# |
|
||||||
# * Redistributions of source code must retain the above copyright |
|
||||||
# notice, this list of conditions and the following disclaimer. |
|
||||||
# * Redistributions in binary form must reproduce the above |
|
||||||
# copyright notice, this list of conditions and the following disclaimer |
|
||||||
# in the documentation and/or other materials provided with the |
|
||||||
# distribution. |
|
||||||
# * Neither the name of Google Inc. nor the names of its |
|
||||||
# contributors may be used to endorse or promote products derived from |
|
||||||
# this software without specific prior written permission. |
|
||||||
# |
|
||||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
||||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
||||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
||||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
||||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
||||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
||||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
||||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
||||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
||||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
||||||
|
|
||||||
from __future__ import print_function |
|
||||||
|
|
||||||
import argparse |
|
||||||
import datetime |
|
||||||
import json |
|
||||||
import os |
|
||||||
import subprocess |
|
||||||
import sys |
|
||||||
import time |
|
||||||
|
|
||||||
stress_test_utils_dir = os.path.abspath(os.path.join( |
|
||||||
os.path.dirname(__file__), '../../gcp/stress_test')) |
|
||||||
sys.path.append(stress_test_utils_dir) |
|
||||||
from stress_test_utils import BigQueryHelper |
|
||||||
|
|
||||||
kubernetes_api_dir = os.path.abspath(os.path.join( |
|
||||||
os.path.dirname(__file__), '../../gcp/utils')) |
|
||||||
sys.path.append(kubernetes_api_dir) |
|
||||||
|
|
||||||
import kubernetes_api |
|
||||||
|
|
||||||
|
|
||||||
class GlobalSettings: |
|
||||||
|
|
||||||
def __init__(self, gcp_project_id, build_docker_images, |
|
||||||
test_poll_interval_secs, test_duration_secs, |
|
||||||
kubernetes_proxy_port, dataset_id_prefix, summary_table_id, |
|
||||||
qps_table_id, pod_warmup_secs): |
|
||||||
self.gcp_project_id = gcp_project_id |
|
||||||
self.build_docker_images = build_docker_images |
|
||||||
self.test_poll_interval_secs = test_poll_interval_secs |
|
||||||
self.test_duration_secs = test_duration_secs |
|
||||||
self.kubernetes_proxy_port = kubernetes_proxy_port |
|
||||||
self.dataset_id_prefix = dataset_id_prefix |
|
||||||
self.summary_table_id = summary_table_id |
|
||||||
self.qps_table_id = qps_table_id |
|
||||||
self.pod_warmup_secs = pod_warmup_secs |
|
||||||
|
|
||||||
|
|
||||||
class ClientTemplate: |
|
||||||
""" Contains all the common settings that are used by a stress client """ |
|
||||||
|
|
||||||
def __init__(self, name, stress_client_cmd, metrics_client_cmd, metrics_port, |
|
||||||
wrapper_script_path, poll_interval_secs, client_args_dict, |
|
||||||
metrics_args_dict, will_run_forever, env_dict): |
|
||||||
self.name = name |
|
||||||
self.stress_client_cmd = stress_client_cmd |
|
||||||
self.metrics_client_cmd = metrics_client_cmd |
|
||||||
self.metrics_port = metrics_port |
|
||||||
self.wrapper_script_path = wrapper_script_path |
|
||||||
self.poll_interval_secs = poll_interval_secs |
|
||||||
self.client_args_dict = client_args_dict |
|
||||||
self.metrics_args_dict = metrics_args_dict |
|
||||||
self.will_run_forever = will_run_forever |
|
||||||
self.env_dict = env_dict |
|
||||||
|
|
||||||
|
|
||||||
class ServerTemplate: |
|
||||||
""" Contains all the common settings used by a stress server """ |
|
||||||
|
|
||||||
def __init__(self, name, server_cmd, wrapper_script_path, server_port, |
|
||||||
server_args_dict, will_run_forever, env_dict): |
|
||||||
self.name = name |
|
||||||
self.server_cmd = server_cmd |
|
||||||
self.wrapper_script_path = wrapper_script_path |
|
||||||
self.server_port = server_port |
|
||||||
self.server_args_dict = server_args_dict |
|
||||||
self.will_run_forever = will_run_forever |
|
||||||
self.env_dict = env_dict |
|
||||||
|
|
||||||
|
|
||||||
class DockerImage: |
|
||||||
""" Represents properties of a Docker image. Provides methods to build the |
|
||||||
image and push it to GKE registry |
|
||||||
""" |
|
||||||
|
|
||||||
def __init__(self, gcp_project_id, image_name, build_script_path, |
|
||||||
dockerfile_dir, build_type): |
|
||||||
"""Args: |
|
||||||
|
|
||||||
image_name: The docker image name |
|
||||||
tag_name: The additional tag name. This is the name used when pushing the |
|
||||||
docker image to GKE registry |
|
||||||
build_script_path: The path to the build script that builds this docker |
|
||||||
image |
|
||||||
dockerfile_dir: The name of the directory under |
|
||||||
'<grpc_root>/tools/dockerfile' that contains the dockerfile |
|
||||||
""" |
|
||||||
self.image_name = image_name |
|
||||||
self.gcp_project_id = gcp_project_id |
|
||||||
self.build_script_path = build_script_path |
|
||||||
self.dockerfile_dir = dockerfile_dir |
|
||||||
self.build_type = build_type |
|
||||||
self.tag_name = self._make_tag_name(gcp_project_id, image_name) |
|
||||||
|
|
||||||
def _make_tag_name(self, project_id, image_name): |
|
||||||
return 'gcr.io/%s/%s' % (project_id, image_name) |
|
||||||
|
|
||||||
def build_image(self): |
|
||||||
print('Building docker image: %s (tag: %s)' % (self.image_name, |
|
||||||
self.tag_name)) |
|
||||||
os.environ['INTEROP_IMAGE'] = self.image_name |
|
||||||
os.environ['INTEROP_IMAGE_REPOSITORY_TAG'] = self.tag_name |
|
||||||
os.environ['BASE_NAME'] = self.dockerfile_dir |
|
||||||
os.environ['BUILD_TYPE'] = self.build_type |
|
||||||
print('DEBUG: path: ', self.build_script_path) |
|
||||||
if subprocess.call(args=[self.build_script_path]) != 0: |
|
||||||
print('Error in building the Docker image') |
|
||||||
return False |
|
||||||
return True |
|
||||||
|
|
||||||
def push_to_gke_registry(self): |
|
||||||
cmd = ['gcloud', 'docker', 'push', self.tag_name] |
|
||||||
print('Pushing %s to the GKE registry..' % self.tag_name) |
|
||||||
if subprocess.call(args=cmd) != 0: |
|
||||||
print('Error in pushing the image %s to the GKE registry' % |
|
||||||
self.tag_name) |
|
||||||
return False |
|
||||||
return True |
|
||||||
|
|
||||||
|
|
||||||
class ServerPodSpec: |
|
||||||
""" Contains the information required to launch server pods. """ |
|
||||||
|
|
||||||
def __init__(self, name, server_template, docker_image, num_instances): |
|
||||||
self.name = name |
|
||||||
self.template = server_template |
|
||||||
self.docker_image = docker_image |
|
||||||
self.num_instances = num_instances |
|
||||||
|
|
||||||
def pod_names(self): |
|
||||||
""" Return a list of names of server pods to create. """ |
|
||||||
return ['%s-%d' % (self.name, i) for i in range(1, self.num_instances + 1)] |
|
||||||
|
|
||||||
def server_addresses(self): |
|
||||||
""" Return string of server addresses in the following format: |
|
||||||
'<server_pod_name_1>:<server_port>,<server_pod_name_2>:<server_port>...' |
|
||||||
""" |
|
||||||
return ','.join(['%s:%d' % (pod_name, self.template.server_port) |
|
||||||
for pod_name in self.pod_names()]) |
|
||||||
|
|
||||||
|
|
||||||
class ClientPodSpec: |
|
||||||
""" Contains the information required to launch client pods """ |
|
||||||
|
|
||||||
def __init__(self, name, client_template, docker_image, num_instances, |
|
||||||
server_addresses): |
|
||||||
self.name = name |
|
||||||
self.template = client_template |
|
||||||
self.docker_image = docker_image |
|
||||||
self.num_instances = num_instances |
|
||||||
self.server_addresses = server_addresses |
|
||||||
|
|
||||||
def pod_names(self): |
|
||||||
""" Return a list of names of client pods to create """ |
|
||||||
return ['%s-%d' % (self.name, i) for i in range(1, self.num_instances + 1)] |
|
||||||
|
|
||||||
# The client args in the template do not have server addresses. This function |
|
||||||
# adds the server addresses and returns the updated client args |
|
||||||
def get_client_args_dict(self): |
|
||||||
args_dict = self.template.client_args_dict.copy() |
|
||||||
args_dict['server_addresses'] = self.server_addresses |
|
||||||
return args_dict |
|
||||||
|
|
||||||
|
|
||||||
class Gke: |
|
||||||
""" Class that has helper methods to interact with GKE """ |
|
||||||
|
|
||||||
class KubernetesProxy: |
|
||||||
"""Class to start a proxy on localhost to talk to the Kubernetes API server""" |
|
||||||
|
|
||||||
def __init__(self, port): |
|
||||||
cmd = ['kubectl', 'proxy', '--port=%d' % port] |
|
||||||
self.p = subprocess.Popen(args=cmd) |
|
||||||
time.sleep(2) |
|
||||||
print('\nStarted kubernetes proxy on port: %d' % port) |
|
||||||
|
|
||||||
def __del__(self): |
|
||||||
if self.p is not None: |
|
||||||
print('Shutting down Kubernetes proxy..') |
|
||||||
self.p.kill() |
|
||||||
|
|
||||||
def __init__(self, project_id, run_id, dataset_id, summary_table_id, |
|
||||||
qps_table_id, kubernetes_port): |
|
||||||
self.project_id = project_id |
|
||||||
self.run_id = run_id |
|
||||||
self.dataset_id = dataset_id |
|
||||||
self.summary_table_id = summary_table_id |
|
||||||
self.qps_table_id = qps_table_id |
|
||||||
|
|
||||||
# The environment variables we would like to pass to every pod (both client |
|
||||||
# and server) launched in GKE |
|
||||||
self.gke_env = { |
|
||||||
'RUN_ID': self.run_id, |
|
||||||
'GCP_PROJECT_ID': self.project_id, |
|
||||||
'DATASET_ID': self.dataset_id, |
|
||||||
'SUMMARY_TABLE_ID': self.summary_table_id, |
|
||||||
'QPS_TABLE_ID': self.qps_table_id |
|
||||||
} |
|
||||||
|
|
||||||
self.kubernetes_port = kubernetes_port |
|
||||||
# Start kubernetes proxy |
|
||||||
self.kubernetes_proxy = Gke.KubernetesProxy(kubernetes_port) |
|
||||||
|
|
||||||
def _args_dict_to_str(self, args_dict): |
|
||||||
return ' '.join('--%s=%s' % (k, args_dict[k]) for k in args_dict.keys()) |
|
||||||
|
|
||||||
def launch_servers(self, server_pod_spec): |
|
||||||
is_success = True |
|
||||||
|
|
||||||
# The command to run inside the container is the wrapper script (which then |
|
||||||
# launches the actual server) |
|
||||||
container_cmd = server_pod_spec.template.wrapper_script_path |
|
||||||
|
|
||||||
# The parameters to the wrapper script (defined in |
|
||||||
# server_pod_spec.template.wrapper_script_path) are are injected into the |
|
||||||
# container via environment variables |
|
||||||
server_env = self.gke_env.copy() |
|
||||||
server_env.update(server_pod_spec.template.env_dict) |
|
||||||
server_env.update({ |
|
||||||
'STRESS_TEST_IMAGE_TYPE': 'SERVER', |
|
||||||
'STRESS_TEST_CMD': server_pod_spec.template.server_cmd, |
|
||||||
'STRESS_TEST_ARGS_STR': self._args_dict_to_str( |
|
||||||
server_pod_spec.template.server_args_dict), |
|
||||||
'WILL_RUN_FOREVER': str(server_pod_spec.template.will_run_forever) |
|
||||||
}) |
|
||||||
|
|
||||||
for pod_name in server_pod_spec.pod_names(): |
|
||||||
server_env['POD_NAME'] = pod_name |
|
||||||
print('Creating server: %s' % pod_name) |
|
||||||
is_success = kubernetes_api.create_pod_and_service( |
|
||||||
'localhost', |
|
||||||
self.kubernetes_port, |
|
||||||
'default', # Use 'default' namespace |
|
||||||
pod_name, |
|
||||||
server_pod_spec.docker_image.tag_name, |
|
||||||
[server_pod_spec.template.server_port], # Ports to expose on the pod |
|
||||||
[container_cmd], |
|
||||||
[], # Args list is empty since we are passing all args via env variables |
|
||||||
server_env, |
|
||||||
True # Headless = True for server to that GKE creates a DNS record for pod_name |
|
||||||
) |
|
||||||
if not is_success: |
|
||||||
print('Error in launching server: %s' % pod_name) |
|
||||||
break |
|
||||||
|
|
||||||
if is_success: |
|
||||||
print('Successfully created server(s)') |
|
||||||
|
|
||||||
return is_success |
|
||||||
|
|
||||||
def launch_clients(self, client_pod_spec): |
|
||||||
is_success = True |
|
||||||
|
|
||||||
# The command to run inside the container is the wrapper script (which then |
|
||||||
# launches the actual stress client) |
|
||||||
container_cmd = client_pod_spec.template.wrapper_script_path |
|
||||||
|
|
||||||
# The parameters to the wrapper script (defined in |
|
||||||
# client_pod_spec.template.wrapper_script_path) are are injected into the |
|
||||||
# container via environment variables |
|
||||||
client_env = self.gke_env.copy() |
|
||||||
client_env.update(client_pod_spec.template.env_dict) |
|
||||||
client_env.update({ |
|
||||||
'STRESS_TEST_IMAGE_TYPE': 'CLIENT', |
|
||||||
'STRESS_TEST_CMD': client_pod_spec.template.stress_client_cmd, |
|
||||||
'STRESS_TEST_ARGS_STR': self._args_dict_to_str( |
|
||||||
client_pod_spec.get_client_args_dict()), |
|
||||||
'METRICS_CLIENT_CMD': client_pod_spec.template.metrics_client_cmd, |
|
||||||
'METRICS_CLIENT_ARGS_STR': self._args_dict_to_str( |
|
||||||
client_pod_spec.template.metrics_args_dict), |
|
||||||
'POLL_INTERVAL_SECS': str(client_pod_spec.template.poll_interval_secs), |
|
||||||
'WILL_RUN_FOREVER': str(client_pod_spec.template.will_run_forever) |
|
||||||
}) |
|
||||||
|
|
||||||
for pod_name in client_pod_spec.pod_names(): |
|
||||||
client_env['POD_NAME'] = pod_name |
|
||||||
print('Creating client: %s' % pod_name) |
|
||||||
is_success = kubernetes_api.create_pod_and_service( |
|
||||||
'localhost', |
|
||||||
self.kubernetes_port, |
|
||||||
'default', # default namespace, |
|
||||||
pod_name, |
|
||||||
client_pod_spec.docker_image.tag_name, |
|
||||||
[client_pod_spec.template.metrics_port], # Ports to expose on the pod |
|
||||||
[container_cmd], |
|
||||||
[], # Empty args list since all args are passed via env variables |
|
||||||
client_env, |
|
||||||
True # Client is a headless service (no need for an external ip) |
|
||||||
) |
|
||||||
|
|
||||||
if not is_success: |
|
||||||
print('Error in launching client %s' % pod_name) |
|
||||||
break |
|
||||||
|
|
||||||
if is_success: |
|
||||||
print('Successfully created all client(s)') |
|
||||||
|
|
||||||
return is_success |
|
||||||
|
|
||||||
def _delete_pods(self, pod_name_list): |
|
||||||
is_success = True |
|
||||||
for pod_name in pod_name_list: |
|
||||||
print('Deleting %s' % pod_name) |
|
||||||
is_success = kubernetes_api.delete_pod_and_service( |
|
||||||
'localhost', |
|
||||||
self.kubernetes_port, |
|
||||||
'default', # default namespace |
|
||||||
pod_name) |
|
||||||
|
|
||||||
if not is_success: |
|
||||||
print('Error in deleting pod %s' % pod_name) |
|
||||||
break |
|
||||||
|
|
||||||
if is_success: |
|
||||||
print('Successfully deleted all pods') |
|
||||||
|
|
||||||
return is_success |
|
||||||
|
|
||||||
def delete_servers(self, server_pod_spec): |
|
||||||
return self._delete_pods(server_pod_spec.pod_names()) |
|
||||||
|
|
||||||
def delete_clients(self, client_pod_spec): |
|
||||||
return self._delete_pods(client_pod_spec.pod_names()) |
|
||||||
|
|
||||||
|
|
||||||
class Config: |
|
||||||
|
|
||||||
def __init__(self, config_filename, gcp_project_id): |
|
||||||
print('Loading configuration...') |
|
||||||
config_dict = self._load_config(config_filename) |
|
||||||
|
|
||||||
self.global_settings = self._parse_global_settings(config_dict, |
|
||||||
gcp_project_id) |
|
||||||
self.docker_images_dict = self._parse_docker_images( |
|
||||||
config_dict, self.global_settings.gcp_project_id) |
|
||||||
self.client_templates_dict = self._parse_client_templates(config_dict) |
|
||||||
self.server_templates_dict = self._parse_server_templates(config_dict) |
|
||||||
self.server_pod_specs_dict = self._parse_server_pod_specs( |
|
||||||
config_dict, self.docker_images_dict, self.server_templates_dict) |
|
||||||
self.client_pod_specs_dict = self._parse_client_pod_specs( |
|
||||||
config_dict, self.docker_images_dict, self.client_templates_dict, |
|
||||||
self.server_pod_specs_dict) |
|
||||||
print('Loaded Configuaration.') |
|
||||||
|
|
||||||
def _parse_global_settings(self, config_dict, gcp_project_id): |
|
||||||
global_settings_dict = config_dict['globalSettings'] |
|
||||||
return GlobalSettings(gcp_project_id, |
|
||||||
global_settings_dict['buildDockerImages'], |
|
||||||
global_settings_dict['pollIntervalSecs'], |
|
||||||
global_settings_dict['testDurationSecs'], |
|
||||||
global_settings_dict['kubernetesProxyPort'], |
|
||||||
global_settings_dict['datasetIdNamePrefix'], |
|
||||||
global_settings_dict['summaryTableId'], |
|
||||||
global_settings_dict['qpsTableId'], |
|
||||||
global_settings_dict['podWarmupSecs']) |
|
||||||
|
|
||||||
def _parse_docker_images(self, config_dict, gcp_project_id): |
|
||||||
"""Parses the 'dockerImages' section of the config file and returns a |
|
||||||
Dictionary of 'DockerImage' objects keyed by docker image names""" |
|
||||||
docker_images_dict = {} |
|
||||||
|
|
||||||
docker_config_dict = config_dict['dockerImages'] |
|
||||||
for image_name in docker_config_dict.keys(): |
|
||||||
build_script_path = docker_config_dict[image_name]['buildScript'] |
|
||||||
dockerfile_dir = docker_config_dict[image_name]['dockerFileDir'] |
|
||||||
build_type = docker_config_dict[image_name].get('buildType', 'opt') |
|
||||||
docker_images_dict[image_name] = DockerImage(gcp_project_id, image_name, |
|
||||||
build_script_path, |
|
||||||
dockerfile_dir, build_type) |
|
||||||
return docker_images_dict |
|
||||||
|
|
||||||
def _parse_client_templates(self, config_dict): |
|
||||||
"""Parses the 'clientTemplates' section of the config file and returns a |
|
||||||
Dictionary of 'ClientTemplate' objects keyed by client template names. |
|
||||||
|
|
||||||
Note: The 'baseTemplates' sub section of the config file contains templates |
|
||||||
with default values and the 'templates' sub section contains the actual |
|
||||||
client templates (which refer to the base template name to use for default |
|
||||||
values). |
|
||||||
""" |
|
||||||
client_templates_dict = {} |
|
||||||
|
|
||||||
templates_dict = config_dict['clientTemplates']['templates'] |
|
||||||
base_templates_dict = config_dict['clientTemplates'].get('baseTemplates', |
|
||||||
{}) |
|
||||||
for template_name in templates_dict.keys(): |
|
||||||
# temp_dict is a temporary dictionary that merges base template dictionary |
|
||||||
# and client template dictionary (with client template dictionary values |
|
||||||
# overriding base template values) |
|
||||||
temp_dict = {} |
|
||||||
|
|
||||||
base_template_name = templates_dict[template_name].get('baseTemplate') |
|
||||||
if not base_template_name is None: |
|
||||||
temp_dict = base_templates_dict[base_template_name].copy() |
|
||||||
|
|
||||||
temp_dict.update(templates_dict[template_name]) |
|
||||||
|
|
||||||
# Create and add ClientTemplate object to the final client_templates_dict |
|
||||||
stress_client_cmd = ' '.join(temp_dict['stressClientCmd']) |
|
||||||
metrics_client_cmd = ' '.join(temp_dict['metricsClientCmd']) |
|
||||||
client_templates_dict[template_name] = ClientTemplate( |
|
||||||
template_name, stress_client_cmd, metrics_client_cmd, |
|
||||||
temp_dict['metricsPort'], temp_dict['wrapperScriptPath'], |
|
||||||
temp_dict['pollIntervalSecs'], temp_dict['clientArgs'].copy(), |
|
||||||
temp_dict['metricsArgs'].copy(), temp_dict.get('willRunForever', 1), |
|
||||||
temp_dict.get('env', {}).copy()) |
|
||||||
|
|
||||||
return client_templates_dict |
|
||||||
|
|
||||||
def _parse_server_templates(self, config_dict): |
|
||||||
"""Parses the 'serverTemplates' section of the config file and returns a |
|
||||||
Dictionary of 'serverTemplate' objects keyed by server template names. |
|
||||||
|
|
||||||
Note: The 'baseTemplates' sub section of the config file contains templates |
|
||||||
with default values and the 'templates' sub section contains the actual |
|
||||||
server templates (which refer to the base template name to use for default |
|
||||||
values). |
|
||||||
""" |
|
||||||
server_templates_dict = {} |
|
||||||
|
|
||||||
templates_dict = config_dict['serverTemplates']['templates'] |
|
||||||
base_templates_dict = config_dict['serverTemplates'].get('baseTemplates', |
|
||||||
{}) |
|
||||||
|
|
||||||
for template_name in templates_dict.keys(): |
|
||||||
# temp_dict is a temporary dictionary that merges base template dictionary |
|
||||||
# and server template dictionary (with server template dictionary values |
|
||||||
# overriding base template values) |
|
||||||
temp_dict = {} |
|
||||||
|
|
||||||
base_template_name = templates_dict[template_name].get('baseTemplate') |
|
||||||
if not base_template_name is None: |
|
||||||
temp_dict = base_templates_dict[base_template_name].copy() |
|
||||||
|
|
||||||
temp_dict.update(templates_dict[template_name]) |
|
||||||
|
|
||||||
# Create and add ServerTemplate object to the final server_templates_dict |
|
||||||
stress_server_cmd = ' '.join(temp_dict['stressServerCmd']) |
|
||||||
server_templates_dict[template_name] = ServerTemplate( |
|
||||||
template_name, stress_server_cmd, temp_dict['wrapperScriptPath'], |
|
||||||
temp_dict['serverPort'], temp_dict['serverArgs'].copy(), |
|
||||||
temp_dict.get('willRunForever', 1), temp_dict.get('env', {}).copy()) |
|
||||||
|
|
||||||
return server_templates_dict |
|
||||||
|
|
||||||
def _parse_server_pod_specs(self, config_dict, docker_images_dict, |
|
||||||
server_templates_dict): |
|
||||||
"""Parses the 'serverPodSpecs' sub-section (under 'testMatrix' section) of |
|
||||||
the config file and returns a Dictionary of 'ServerPodSpec' objects keyed |
|
||||||
by server pod spec names""" |
|
||||||
server_pod_specs_dict = {} |
|
||||||
|
|
||||||
pod_specs_dict = config_dict['testMatrix'].get('serverPodSpecs', {}) |
|
||||||
|
|
||||||
for pod_name in pod_specs_dict.keys(): |
|
||||||
server_template_name = pod_specs_dict[pod_name]['serverTemplate'] |
|
||||||
docker_image_name = pod_specs_dict[pod_name]['dockerImage'] |
|
||||||
num_instances = pod_specs_dict[pod_name].get('numInstances', 1) |
|
||||||
|
|
||||||
# Create and add the ServerPodSpec object to the final |
|
||||||
# server_pod_specs_dict |
|
||||||
server_pod_specs_dict[pod_name] = ServerPodSpec( |
|
||||||
pod_name, server_templates_dict[server_template_name], |
|
||||||
docker_images_dict[docker_image_name], num_instances) |
|
||||||
|
|
||||||
return server_pod_specs_dict |
|
||||||
|
|
||||||
def _parse_client_pod_specs(self, config_dict, docker_images_dict, |
|
||||||
client_templates_dict, server_pod_specs_dict): |
|
||||||
"""Parses the 'clientPodSpecs' sub-section (under 'testMatrix' section) of |
|
||||||
the config file and returns a Dictionary of 'ClientPodSpec' objects keyed |
|
||||||
by client pod spec names""" |
|
||||||
client_pod_specs_dict = {} |
|
||||||
|
|
||||||
pod_specs_dict = config_dict['testMatrix'].get('clientPodSpecs', {}) |
|
||||||
for pod_name in pod_specs_dict.keys(): |
|
||||||
client_template_name = pod_specs_dict[pod_name]['clientTemplate'] |
|
||||||
docker_image_name = pod_specs_dict[pod_name]['dockerImage'] |
|
||||||
num_instances = pod_specs_dict[pod_name]['numInstances'] |
|
||||||
|
|
||||||
# Get the server addresses from the server pod spec object |
|
||||||
server_pod_spec_name = pod_specs_dict[pod_name]['serverPodSpec'] |
|
||||||
server_addresses = server_pod_specs_dict[ |
|
||||||
server_pod_spec_name].server_addresses() |
|
||||||
|
|
||||||
client_pod_specs_dict[pod_name] = ClientPodSpec( |
|
||||||
pod_name, client_templates_dict[client_template_name], |
|
||||||
docker_images_dict[docker_image_name], num_instances, |
|
||||||
server_addresses) |
|
||||||
|
|
||||||
return client_pod_specs_dict |
|
||||||
|
|
||||||
def _load_config(self, config_filename): |
|
||||||
"""Opens the config file and converts the Json text to Dictionary""" |
|
||||||
if not os.path.isabs(config_filename): |
|
||||||
raise Exception('Config objects expects an absolute file path. ' |
|
||||||
'config file name passed: %s' % config_filename) |
|
||||||
with open(config_filename) as config_file: |
|
||||||
return json.load(config_file) |
|
||||||
|
|
||||||
|
|
||||||
def run_tests(config): |
|
||||||
""" The main function that launches the stress tests """ |
|
||||||
# Build docker images and push to GKE registry |
|
||||||
if config.global_settings.build_docker_images: |
|
||||||
for name, docker_image in config.docker_images_dict.iteritems(): |
|
||||||
if not (docker_image.build_image() and |
|
||||||
docker_image.push_to_gke_registry()): |
|
||||||
return False |
|
||||||
|
|
||||||
# Create a unique id for this run (Note: Using timestamp instead of UUID to |
|
||||||
# make it easier to deduce the date/time of the run just by looking at the run |
|
||||||
# run id. This is useful in debugging when looking at records in Biq query) |
|
||||||
run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S') |
|
||||||
dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id) |
|
||||||
print('Run id:', run_id) |
|
||||||
print('Dataset id:', dataset_id) |
|
||||||
|
|
||||||
bq_helper = BigQueryHelper(run_id, '', '', |
|
||||||
config.global_settings.gcp_project_id, dataset_id, |
|
||||||
config.global_settings.summary_table_id, |
|
||||||
config.global_settings.qps_table_id) |
|
||||||
bq_helper.initialize() |
|
||||||
|
|
||||||
gke = Gke(config.global_settings.gcp_project_id, run_id, dataset_id, |
|
||||||
config.global_settings.summary_table_id, |
|
||||||
config.global_settings.qps_table_id, |
|
||||||
config.global_settings.kubernetes_proxy_port) |
|
||||||
|
|
||||||
is_success = True |
|
||||||
|
|
||||||
try: |
|
||||||
print('Launching servers..') |
|
||||||
for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): |
|
||||||
if not gke.launch_servers(server_pod_spec): |
|
||||||
is_success = False # is_success is checked in the 'finally' block |
|
||||||
return False |
|
||||||
|
|
||||||
print('Launched servers. Waiting for %d seconds for the server pods to be ' |
|
||||||
'fully online') % config.global_settings.pod_warmup_secs |
|
||||||
time.sleep(config.global_settings.pod_warmup_secs) |
|
||||||
|
|
||||||
for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): |
|
||||||
if not gke.launch_clients(client_pod_spec): |
|
||||||
is_success = False # is_success is checked in the 'finally' block |
|
||||||
return False |
|
||||||
|
|
||||||
print('Launched all clients. Waiting for %d seconds for the client pods to ' |
|
||||||
'be fully online') % config.global_settings.pod_warmup_secs |
|
||||||
time.sleep(config.global_settings.pod_warmup_secs) |
|
||||||
|
|
||||||
start_time = datetime.datetime.now() |
|
||||||
end_time = start_time + datetime.timedelta( |
|
||||||
seconds=config.global_settings.test_duration_secs) |
|
||||||
print('Running the test until %s' % end_time.isoformat()) |
|
||||||
|
|
||||||
while True: |
|
||||||
if datetime.datetime.now() > end_time: |
|
||||||
print('Test was run for %d seconds' % |
|
||||||
config.global_settings.test_duration_secs) |
|
||||||
break |
|
||||||
|
|
||||||
# Check if either stress server or clients have failed (btw, the bq_helper |
|
||||||
# monitors all the rows in the summary table and checks if any of them |
|
||||||
# have a failure status) |
|
||||||
if bq_helper.check_if_any_tests_failed(): |
|
||||||
is_success = False |
|
||||||
print('Some tests failed.') |
|
||||||
break # Don't 'return' here. We still want to call bq_helper to print qps/summary tables |
|
||||||
|
|
||||||
# Tests running fine. Wait until next poll time to check the status |
|
||||||
print('Sleeping for %d seconds..' % |
|
||||||
config.global_settings.test_poll_interval_secs) |
|
||||||
time.sleep(config.global_settings.test_poll_interval_secs) |
|
||||||
|
|
||||||
# Print BiqQuery tables |
|
||||||
bq_helper.print_qps_records() |
|
||||||
bq_helper.print_summary_records() |
|
||||||
|
|
||||||
finally: |
|
||||||
# If there was a test failure, we should not delete the pods since they |
|
||||||
# would contain useful debug information (logs, core dumps etc) |
|
||||||
if is_success: |
|
||||||
for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): |
|
||||||
gke.delete_servers(server_pod_spec) |
|
||||||
for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): |
|
||||||
gke.delete_clients(client_pod_spec) |
|
||||||
|
|
||||||
return is_success |
|
||||||
|
|
||||||
|
|
||||||
def tear_down(config): |
|
||||||
gke = Gke(config.global_settings.gcp_project_id, '', '', |
|
||||||
config.global_settings.summary_table_id, |
|
||||||
config.global_settings.qps_table_id, |
|
||||||
config.global_settings.kubernetes_proxy_port) |
|
||||||
for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): |
|
||||||
gke.delete_servers(server_pod_spec) |
|
||||||
for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): |
|
||||||
gke.delete_clients(client_pod_spec) |
|
||||||
|
|
||||||
|
|
||||||
argp = argparse.ArgumentParser( |
|
||||||
description='Launch stress tests in GKE', |
|
||||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
|
||||||
argp.add_argument('--gcp_project_id', |
|
||||||
required=True, |
|
||||||
help='The Google Cloud Platform Project Id') |
|
||||||
argp.add_argument('--config_file', |
|
||||||
required=True, |
|
||||||
type=str, |
|
||||||
help='The test config file') |
|
||||||
argp.add_argument('--tear_down', action='store_true', default=False) |
|
||||||
|
|
||||||
if __name__ == '__main__': |
|
||||||
args = argp.parse_args() |
|
||||||
|
|
||||||
config_filename = args.config_file |
|
||||||
|
|
||||||
# Since we will be changing the current working directory to grpc root in the |
|
||||||
# next step, we should check if the config filename path is a relative path |
|
||||||
# (i.e a path relative to the current working directory) and if so, convert it |
|
||||||
# to abosulte path |
|
||||||
if not os.path.isabs(config_filename): |
|
||||||
config_filename = os.path.abspath(config_filename) |
|
||||||
|
|
||||||
config = Config(config_filename, args.gcp_project_id) |
|
||||||
|
|
||||||
# Change current working directory to grpc root |
|
||||||
# (This is important because all relative file paths in the config file are |
|
||||||
# supposed to interpreted as relative to the GRPC root) |
|
||||||
grpc_root = os.path.abspath(os.path.join( |
|
||||||
os.path.dirname(sys.argv[0]), '../../..')) |
|
||||||
os.chdir(grpc_root) |
|
||||||
|
|
||||||
# Note that tear_down is only in cases where we want to manually tear down a |
|
||||||
# test that for some reason run_tests() could not cleanup |
|
||||||
if args.tear_down: |
|
||||||
tear_down(config) |
|
||||||
sys.exit(1) |
|
||||||
|
|
||||||
if not run_tests(config): |
|
||||||
sys.exit(1) |
|
||||||
Loading…
Reference in new issue