Merge pull request #15202 from mehrdada/python-source-distrib-tests

Python source distribtests

test/distrib/python/test_packages.sh

@@ -22,12 +22,14 @@ shopt -s nullglob
 if [[ "$1" == "binary" ]]
 then
   echo "Testing Python binary distribution"
-  ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-*.whl)
-  TOOLS_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio_tools-*.whl)
+  ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-[0-9]*.whl)
+  TOOLS_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio_tools-[0-9]*.whl)
 else
   echo "Testing Python source distribution"
-  ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-*.tar.gz)
-  TOOLS_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio_tools-*.tar.gz)
+  ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-[0-9]*.tar.gz)
+  TOOLS_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-tools-[0-9]*.tar.gz)
+  HEALTH_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-health-checking-[0-9]*.tar.gz)
+  REFLECTION_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-reflection-[0-9]*.tar.gz)
 fi
 
 VIRTUAL_ENV=$(mktemp -d)
@@ -52,6 +54,14 @@ function at_least_one_installs() {
 at_least_one_installs "${ARCHIVES[@]}"
 at_least_one_installs "${TOOLS_ARCHIVES[@]}"
 
+if [[ "$1" == "source" ]]
+then
+  echo "Testing Python health and reflection packages"
+  at_least_one_installs "${HEALTH_ARCHIVES[@]}"
+  at_least_one_installs "${REFLECTION_ARCHIVES[@]}"
+fi
+
+
 #
 # Test our distributions
 #

tools/distrib/yapf_code.sh

@@ -30,11 +30,11 @@ EXCLUSIONS=(
 
 VIRTUALENV=yapf_virtual_environment
 
-virtualenv $VIRTUALENV
-PYTHON=$(realpath "${VIRTUALENV}/bin/python")
-$PYTHON -m pip install --upgrade pip==9.0.2
-$PYTHON -m pip install --upgrade futures
-$PYTHON -m pip install yapf==0.20.0
+python -m virtualenv $VIRTUALENV
+PYTHON=${VIRTUALENV}/bin/python
+"$PYTHON" -m pip install --upgrade pip==10.0.1
+"$PYTHON" -m pip install --upgrade futures
+"$PYTHON" -m pip install yapf==0.20.0
 
 yapf() {
     local exclusion exclusion_args=()

tools/dockerfile/distribtest/python_dev_arch_x64/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM base/archlinux
+
+RUN pacman --noconfirm -Syy
+RUN pacman --noconfirm -S openssl
+RUN pacman --noconfirm -S python2
+RUN pacman --noconfirm -S python2-pip
+RUN pip2 install virtualenv
+RUN pacman --noconfirm -S base-devel

tools/dockerfile/distribtest/python_dev_centos7_x64/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM centos:7
+
+RUN yum install -y python
+RUN yum install -y epel-release
+RUN yum install -y python-pip
+RUN pip install virtualenv
+RUN yum groupinstall -y 'Development Tools'
+RUN yum install -y python-devel

tools/dockerfile/distribtest/python_dev_fedora22_x64/Dockerfile

@@ -0,0 +1,23 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM fedora:22
+
+RUN yum clean all && yum update -y && yum install -y python python-pip
+RUN pip install virtualenv
+
+RUN yum groupinstall -y "Development Tools"
+RUN yum install -y redhat-rpm-config
+RUN yum install -y gcc-c++
+RUN yum install -y python2-devel

tools/dockerfile/distribtest/python_dev_fedora23_x64/Dockerfile

@@ -0,0 +1,23 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM fedora:23
+
+RUN yum clean all && yum update -y && yum install -y python python-pip
+RUN pip install virtualenv
+
+RUN yum groupinstall -y "Development Tools"
+RUN yum install -y redhat-rpm-config
+RUN yum install -y gcc-c++
+RUN yum install -y python2-devel

tools/dockerfile/distribtest/python_dev_jessie_x64/Dockerfile

@@ -0,0 +1,21 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM debian:jessie
+
+RUN apt-get update && apt-get install -y python python-pip
+RUN pip install virtualenv
+
+RUN apt-get install -y build-essential
+RUN apt-get install -y python-dev

tools/dockerfile/distribtest/python_dev_jessie_x86/Dockerfile

@@ -0,0 +1,27 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM 32bit/debian:jessie
+
+RUN apt-get update && apt-get install -y python python-pip
+
+RUN pip install virtualenv
+
+RUN apt-get install -y build-essential
+RUN apt-get install -y python-dev
+
+# docker is running on a 64-bit machine, so we need to
+# override "uname -m" to report i686 instead of x86_64, otherwise
+# python will choose a wrong binary package to install.
+ENTRYPOINT ["linux32"]

tools/dockerfile/distribtest/python_dev_ubuntu1404_x64/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:14.04
+
+RUN apt-get update -y && apt-get install -y python python-pip
+
+RUN apt-get install -y build-essential
+RUN apt-get install -y python-dev
+
+RUN pip install virtualenv

tools/dockerfile/distribtest/python_dev_ubuntu1604_x64/Dockerfile

@@ -0,0 +1,22 @@
+# Copyright 2015 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:16.04
+
+RUN apt-get update -y && apt-get install -y python python-pip
+
+RUN apt-get install -y build-essential
+RUN apt-get install -y python-dev
+
+RUN pip install virtualenv

tools/run_tests/artifacts/distribtest_targets.py

@@ -146,8 +146,12 @@ class CSharpDistribTest(object):
 class PythonDistribTest(object):
     """Tests Python package"""
 
-    def __init__(self, platform, arch, docker_suffix):
-        self.name = 'python_%s_%s_%s' % (platform, arch, docker_suffix)
+    def __init__(self, platform, arch, docker_suffix, source=False):
+        self.source = source
+        if source:
+            self.name = 'python_dev_%s_%s_%s' % (platform, arch, docker_suffix)
+        else:
+            self.name = 'python_%s_%s_%s' % (platform, arch, docker_suffix)
         self.platform = platform
         self.arch = arch
         self.docker_suffix = docker_suffix
@@ -160,12 +164,20 @@ class PythonDistribTest(object):
         if not self.platform == 'linux':
             raise Exception("Not supported yet.")
 
-        return create_docker_jobspec(
-            self.name,
-            'tools/dockerfile/distribtest/python_%s_%s' % (self.docker_suffix,
-                                                           self.arch),
-            'test/distrib/python/run_binary_distrib_test.sh',
-            copy_rel_path='test/distrib')
+        if self.source:
+            return create_docker_jobspec(
+                self.name,
+                'tools/dockerfile/distribtest/python_dev_%s_%s' %
+                (self.docker_suffix, self.arch),
+                'test/distrib/python/run_source_distrib_test.sh',
+                copy_rel_path='test/distrib')
+        else:
+            return create_docker_jobspec(
+                self.name,
+                'tools/dockerfile/distribtest/python_%s_%s' %
+                (self.docker_suffix, self.arch),
+                'test/distrib/python/run_binary_distrib_test.sh',
+                copy_rel_path='test/distrib')
 
     def __str__(self):
         return self.name
@@ -315,6 +327,14 @@ def targets():
         PythonDistribTest('linux', 'x64', 'ubuntu1204'),
         PythonDistribTest('linux', 'x64', 'ubuntu1404'),
         PythonDistribTest('linux', 'x64', 'ubuntu1604'),
+        PythonDistribTest('linux', 'x64', 'jessie', source=True),
+        PythonDistribTest('linux', 'x86', 'jessie', source=True),
+        PythonDistribTest('linux', 'x64', 'centos7', source=True),
+        PythonDistribTest('linux', 'x64', 'fedora22', source=True),
+        PythonDistribTest('linux', 'x64', 'fedora23', source=True),
+        PythonDistribTest('linux', 'x64', 'arch', source=True),
+        PythonDistribTest('linux', 'x64', 'ubuntu1404', source=True),
+        PythonDistribTest('linux', 'x64', 'ubuntu1604', source=True),
         RubyDistribTest('linux', 'x64', 'wheezy'),
         RubyDistribTest('linux', 'x64', 'jessie'),
         RubyDistribTest('linux', 'x86', 'jessie'),