From 18cc842d6fab8a485461a9cfc6f33dc6048f14d1 Mon Sep 17 00:00:00 2001
From: Masood Malekghassemi <soltanmm@users.noreply.github.com>
Date: Fri, 9 Oct 2015 17:55:45 -0700
Subject: [PATCH] Add Python support to run_interop_tests.py
---
tools/jenkins/grpc_interop_python/Dockerfile | 80 +++++++++++++++++++
.../grpc_interop_python/build_interop.sh | 47 +++++++++++
tools/run_tests/run_interop_tests.py | 64 ++++++++++++++-
3 files changed, 187 insertions(+), 4 deletions(-)
create mode 100644 tools/jenkins/grpc_interop_python/Dockerfile
create mode 100755 tools/jenkins/grpc_interop_python/build_interop.sh
diff --git a/tools/jenkins/grpc_interop_python/Dockerfile b/tools/jenkins/grpc_interop_python/Dockerfile
new file mode 100644
index 00000000000..5850f5f321e
--- /dev/null
+++ b/tools/jenkins/grpc_interop_python/Dockerfile
@@ -0,0 +1,80 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# A work-in-progress Dockerfile that allows running gRPC test suites
+# inside a docker container.
+
+FROM debian:jessie
+
+# Install Git.
+RUN apt-get update && apt-get install -y \
+ autoconf \
+ autotools-dev \
+ build-essential \
+ bzip2 \
+ ccache \
+ curl \
+ gcc \
+ gcc-multilib \
+ git \
+ gyp \
+ libc6 \
+ libc6-dbg \
+ libc6-dev \
+ libgtest-dev \
+ libtool \
+ make \
+ strace \
+ python-dev \
+ python-pip \
+ python-setuptools \
+ python-yaml \
+ telnet \
+ unzip \
+ wget \
+ zip && apt-get clean
+
+# Prepare ccache
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+
+#####################
+# Python dependencies
+
+# Install Python requisites
+RUN /bin/bash -l -c "pip install --upgrade pip"
+RUN /bin/bash -l -c "pip install virtualenv"
+
+# Define the default command.
+CMD ["bash"]
diff --git a/tools/jenkins/grpc_interop_python/build_interop.sh b/tools/jenkins/grpc_interop_python/build_interop.sh
new file mode 100755
index 00000000000..8f5bfd11e20
--- /dev/null
+++ b/tools/jenkins/grpc_interop_python/build_interop.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Builds Python interop server and client in a base image.
+set -e
+
+mkdir -p /var/local/git
+git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+
+# copy service account keys if available
+cp -r /var/local/jenkins/service_account $HOME || true
+
+cd /var/local/git/grpc
+
+make install-certs
+make
+
+# build Python interop client and server
+CONFIG=opt ./tools/run_tests/build_python.sh 2.7
+
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 6daa967bba4..33ebcbcc41f 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -56,7 +56,7 @@ _CLOUD_TO_PROD_BASE_ARGS = [
_CLOUD_TO_CLOUD_BASE_ARGS = [
'--server_host_override=foo.test.google.fr']
-# TOOD(jtattermusch) wrapped languages use this variable for location
+# TOOD(jtattermusch) wrapped languages use this variable for location
# of roots.pem. We might want to use GRPC_DEFAULT_SSL_ROOTS_FILE_PATH
# supported by C core SslCredentials instead.
_SSL_CERT_ENV = { 'SSL_CERT_FILE':'/usr/local/share/grpc/roots.pem' }
@@ -87,6 +87,9 @@ class CXXLanguage:
def server_args(self):
return ['bins/opt/interop_server', '--use_tls=true']
+ def global_env(self):
+ return {}
+
def __str__(self):
return 'c++'
@@ -113,6 +116,9 @@ class CSharpLanguage:
def server_args(self):
return ['mono', 'Grpc.IntegrationTesting.Server.exe', '--use_tls=true']
+ def global_env(self):
+ return {}
+
def __str__(self):
return 'csharp'
@@ -139,6 +145,9 @@ class JavaLanguage:
def server_args(self):
return ['./run-test-server.sh', '--use_tls=true']
+ def global_env(self):
+ return {}
+
def __str__(self):
return 'java'
@@ -166,6 +175,9 @@ class GoLanguage:
def server_args(self):
return ['go', 'run', 'server.go', '--use_tls=true']
+ def global_env(self):
+ return {}
+
def __str__(self):
return 'go'
@@ -192,6 +204,9 @@ class NodeLanguage:
def server_args(self):
return ['node', 'src/node/interop/interop_server.js', '--use_tls=true']
+ def global_env(self):
+ return {}
+
def __str__(self):
return 'node'
@@ -214,6 +229,9 @@ class PHPLanguage:
def cloud_to_prod_env(self):
return _SSL_CERT_ENV
+ def global_env(self):
+ return {}
+
def __str__(self):
return 'php'
@@ -240,11 +258,42 @@ class RubyLanguage:
def server_args(self):
return ['ruby', 'src/ruby/bin/interop/interop_server.rb', '--use_tls']
+ def global_env(self):
+ return {}
+
def __str__(self):
return 'ruby'
-# TODO(jtattermusch): python once we get it working
+class PythonLanguage:
+
+ def __init__(self):
+ self.client_cmdline_base = ['python2.7_virtual_environment/bin/python', '-m', 'grpc_interop.client']
+ self.client_cwd = None
+ self.server_cwd = None
+ self.safename = str(self)
+
+ def cloud_to_prod_args(self):
+ return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS +
+ ['--use_tls'])
+
+ def cloud_to_cloud_args(self):
+ return (self.client_cmdline_base + _CLOUD_TO_CLOUD_BASE_ARGS +
+ ['--use_tls', '--use_test_ca'])
+
+ def cloud_to_prod_env(self):
+ return _SSL_CERT_ENV
+
+ def server_args(self):
+ return ['python2.7_virtual_environment/bin/python', '-m', 'grpc_interop.server', '--use_tls']
+
+ def global_env(self):
+ return {'LD_LIBRARY_PATH': 'libs/opt'}
+
+ def __str__(self):
+ return 'python'
+
+
_LANGUAGES = {
'c++' : CXXLanguage(),
'csharp' : CSharpLanguage(),
@@ -253,10 +302,11 @@ _LANGUAGES = {
'node' : NodeLanguage(),
'php' : PHPLanguage(),
'ruby' : RubyLanguage(),
+ 'python' : PythonLanguage(),
}
# languages supported as cloud_to_cloud servers
-_SERVERS = ['c++', 'node', 'csharp', 'java', 'go', 'ruby']
+_SERVERS = ['c++', 'node', 'csharp', 'java', 'go', 'ruby', 'python']
# TODO(jtattermusch): add timeout_on_sleeping_server once java starts supporting it.
_TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong',
@@ -335,7 +385,7 @@ def cloud_to_prod_jobspec(language, test_case, docker_image=None, auth=False):
"""Creates jobspec for cloud-to-prod interop test"""
cmdline = language.cloud_to_prod_args() + ['--test_case=%s' % test_case]
cwd = language.client_cwd
- environ = language.cloud_to_prod_env()
+ environ = dict(language.cloud_to_prod_env(), **language.global_env())
container_name = None
if auth:
cmdline, environ = add_auth_options(language, test_case, cmdline, environ)
@@ -374,10 +424,12 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
'--server_host=%s' % server_host,
'--server_port=%s' % server_port ])
cwd = language.client_cwd
+ environ = language.global_env()
if docker_image:
container_name = dockerjob.random_name('interop_client_%s' % language.safename)
cmdline = docker_run_cmdline(cmdline,
image=docker_image,
+ environ=environ,
cwd=cwd,
docker_args=['--net=host',
'--name', container_name])
@@ -386,6 +438,7 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
test_job = jobset.JobSpec(
cmdline=cmdline,
cwd=cwd,
+ environ=environ,
shortname="cloud_to_cloud:%s:%s_server:%s" % (language, server_name,
test_case),
timeout_seconds=2*60,
@@ -401,13 +454,16 @@ def server_jobspec(language, docker_image):
container_name = dockerjob.random_name('interop_server_%s' % language.safename)
cmdline = bash_login_cmdline(language.server_args() +
['--port=%s' % _DEFAULT_SERVER_PORT])
+ environ = language.global_env()
docker_cmdline = docker_run_cmdline(cmdline,
image=docker_image,
cwd=language.server_cwd,
+ environ=environ,
docker_args=['-p', str(_DEFAULT_SERVER_PORT),
'--name', container_name])
server_job = jobset.JobSpec(
cmdline=docker_cmdline,
+ environ=environ,
shortname="interop_server_%s" % language,
timeout_seconds=30*60)
server_job.container_name = container_name