From bb6d51adaadabdebcbbf439837d053fc411d0a95 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Wed, 25 Apr 2018 10:55:43 -0700 Subject: [PATCH 1/4] Add Dockerfiles for Python source distribtests --- .../python_dev_arch_x64/Dockerfile | 22 +++++++++++++++ .../python_dev_centos7_x64/Dockerfile | 22 +++++++++++++++ .../python_dev_fedora22_x64/Dockerfile | 23 ++++++++++++++++ .../python_dev_fedora23_x64/Dockerfile | 23 ++++++++++++++++ .../python_dev_jessie_x64/Dockerfile | 21 +++++++++++++++ .../python_dev_jessie_x86/Dockerfile | 27 +++++++++++++++++++ .../python_dev_ubuntu1404_x64/Dockerfile | 22 +++++++++++++++ .../python_dev_ubuntu1604_x64/Dockerfile | 22 +++++++++++++++ 8 files changed, 182 insertions(+) create mode 100644 tools/dockerfile/distribtest/python_dev_arch_x64/Dockerfile create mode 100644 tools/dockerfile/distribtest/python_dev_centos7_x64/Dockerfile create mode 100644 tools/dockerfile/distribtest/python_dev_fedora22_x64/Dockerfile create mode 100644 tools/dockerfile/distribtest/python_dev_fedora23_x64/Dockerfile create mode 100644 tools/dockerfile/distribtest/python_dev_jessie_x64/Dockerfile create mode 100644 tools/dockerfile/distribtest/python_dev_jessie_x86/Dockerfile create mode 100644 tools/dockerfile/distribtest/python_dev_ubuntu1404_x64/Dockerfile create mode 100644 tools/dockerfile/distribtest/python_dev_ubuntu1604_x64/Dockerfile diff --git a/tools/dockerfile/distribtest/python_dev_arch_x64/Dockerfile b/tools/dockerfile/distribtest/python_dev_arch_x64/Dockerfile new file mode 100644 index 00000000000..7f09fd64234 --- /dev/null +++ b/tools/dockerfile/distribtest/python_dev_arch_x64/Dockerfile @@ -0,0 +1,22 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM base/archlinux + +RUN pacman --noconfirm -Syy +RUN pacman --noconfirm -S openssl +RUN pacman --noconfirm -S python2 +RUN pacman --noconfirm -S python2-pip +RUN pip2 install virtualenv +RUN pacman --noconfirm -S base-devel diff --git a/tools/dockerfile/distribtest/python_dev_centos7_x64/Dockerfile b/tools/dockerfile/distribtest/python_dev_centos7_x64/Dockerfile new file mode 100644 index 00000000000..954146c5e91 --- /dev/null +++ b/tools/dockerfile/distribtest/python_dev_centos7_x64/Dockerfile @@ -0,0 +1,22 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos:7 + +RUN yum install -y python +RUN yum install -y epel-release +RUN yum install -y python-pip +RUN pip install virtualenv +RUN yum groupinstall -y 'Development Tools' +RUN yum install -y python-devel diff --git a/tools/dockerfile/distribtest/python_dev_fedora22_x64/Dockerfile b/tools/dockerfile/distribtest/python_dev_fedora22_x64/Dockerfile new file mode 100644 index 00000000000..d86ad378c33 --- /dev/null +++ b/tools/dockerfile/distribtest/python_dev_fedora22_x64/Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM fedora:22 + +RUN yum clean all && yum update -y && yum install -y python python-pip +RUN pip install virtualenv + +RUN yum groupinstall -y "Development Tools" +RUN yum install -y redhat-rpm-config +RUN yum install -y gcc-c++ +RUN yum install -y python2-devel diff --git a/tools/dockerfile/distribtest/python_dev_fedora23_x64/Dockerfile b/tools/dockerfile/distribtest/python_dev_fedora23_x64/Dockerfile new file mode 100644 index 00000000000..0dbf5e4c21a --- /dev/null +++ b/tools/dockerfile/distribtest/python_dev_fedora23_x64/Dockerfile @@ -0,0 +1,23 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM fedora:23 + +RUN yum clean all && yum update -y && yum install -y python python-pip +RUN pip install virtualenv + +RUN yum groupinstall -y "Development Tools" +RUN yum install -y redhat-rpm-config +RUN yum install -y gcc-c++ +RUN yum install -y python2-devel diff --git a/tools/dockerfile/distribtest/python_dev_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/python_dev_jessie_x64/Dockerfile new file mode 100644 index 00000000000..c2228989cdc --- /dev/null +++ b/tools/dockerfile/distribtest/python_dev_jessie_x64/Dockerfile @@ -0,0 +1,21 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:jessie + +RUN apt-get update && apt-get install -y python python-pip +RUN pip install virtualenv + +RUN apt-get install -y build-essential +RUN apt-get install -y python-dev diff --git a/tools/dockerfile/distribtest/python_dev_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/python_dev_jessie_x86/Dockerfile new file mode 100644 index 00000000000..5e0b8efe756 --- /dev/null +++ b/tools/dockerfile/distribtest/python_dev_jessie_x86/Dockerfile @@ -0,0 +1,27 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM 32bit/debian:jessie + +RUN apt-get update && apt-get install -y python python-pip + +RUN pip install virtualenv + +RUN apt-get install -y build-essential +RUN apt-get install -y python-dev + +# docker is running on a 64-bit machine, so we need to +# override "uname -m" to report i686 instead of x86_64, otherwise +# python will choose a wrong binary package to install. +ENTRYPOINT ["linux32"] diff --git a/tools/dockerfile/distribtest/python_dev_ubuntu1404_x64/Dockerfile b/tools/dockerfile/distribtest/python_dev_ubuntu1404_x64/Dockerfile new file mode 100644 index 00000000000..6c842aeba00 --- /dev/null +++ b/tools/dockerfile/distribtest/python_dev_ubuntu1404_x64/Dockerfile @@ -0,0 +1,22 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:14.04 + +RUN apt-get update -y && apt-get install -y python python-pip + +RUN apt-get install -y build-essential +RUN apt-get install -y python-dev + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_dev_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/python_dev_ubuntu1604_x64/Dockerfile new file mode 100644 index 00000000000..1ff1e0a1d61 --- /dev/null +++ b/tools/dockerfile/distribtest/python_dev_ubuntu1604_x64/Dockerfile @@ -0,0 +1,22 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:16.04 + +RUN apt-get update -y && apt-get install -y python python-pip + +RUN apt-get install -y build-essential +RUN apt-get install -y python-dev + +RUN pip install virtualenv From 97904d46e06085bb1fa3d3318933d570d0b4468b Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Thu, 26 Apr 2018 15:00:51 -0700 Subject: [PATCH 2/4] Add Python source distribtest targets --- .../artifacts/distribtest_targets.py | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 9087841c4bd..bfb0a9e57fd 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -146,8 +146,12 @@ class CSharpDistribTest(object): class PythonDistribTest(object): """Tests Python package""" - def __init__(self, platform, arch, docker_suffix): - self.name = 'python_%s_%s_%s' % (platform, arch, docker_suffix) + def __init__(self, platform, arch, docker_suffix, source=False): + self.source = source + if source: + self.name = 'python_dev_%s_%s_%s' % (platform, arch, docker_suffix) + else: + self.name = 'python_%s_%s_%s' % (platform, arch, docker_suffix) self.platform = platform self.arch = arch self.docker_suffix = docker_suffix @@ -160,12 +164,20 @@ class PythonDistribTest(object): if not self.platform == 'linux': raise Exception("Not supported yet.") - return create_docker_jobspec( - self.name, - 'tools/dockerfile/distribtest/python_%s_%s' % (self.docker_suffix, - self.arch), - 'test/distrib/python/run_binary_distrib_test.sh', - copy_rel_path='test/distrib') + if self.source: + return create_docker_jobspec( + self.name, + 'tools/dockerfile/distribtest/python_dev_%s_%s' % ( + self.docker_suffix, self.arch), + 'test/distrib/python/run_source_distrib_test.sh', + copy_rel_path='test/distrib') + else: + return create_docker_jobspec( + self.name, + 'tools/dockerfile/distribtest/python_%s_%s' % ( + self.docker_suffix, self.arch), + 'test/distrib/python/run_binary_distrib_test.sh', + copy_rel_path='test/distrib') def __str__(self): return self.name @@ -315,6 +327,14 @@ def targets(): PythonDistribTest('linux', 'x64', 'ubuntu1204'), PythonDistribTest('linux', 'x64', 'ubuntu1404'), PythonDistribTest('linux', 'x64', 'ubuntu1604'), + PythonDistribTest('linux', 'x64', 'jessie', source=True), + PythonDistribTest('linux', 'x86', 'jessie', source=True), + PythonDistribTest('linux', 'x64', 'centos7', source=True), + PythonDistribTest('linux', 'x64', 'fedora22', source=True), + PythonDistribTest('linux', 'x64', 'fedora23', source=True), + PythonDistribTest('linux', 'x64', 'arch', source=True), + PythonDistribTest('linux', 'x64', 'ubuntu1404', source=True), + PythonDistribTest('linux', 'x64', 'ubuntu1604', source=True), RubyDistribTest('linux', 'x64', 'wheezy'), RubyDistribTest('linux', 'x64', 'jessie'), RubyDistribTest('linux', 'x86', 'jessie'), From f4bea02a725bd3743a1a93dcf9adfefc76a9a58b Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Fri, 27 Apr 2018 00:12:13 -0700 Subject: [PATCH 3/4] Fix the Python source distribtest script --- test/distrib/python/test_packages.sh | 18 ++++++++++++++---- .../run_tests/artifacts/distribtest_targets.py | 8 ++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/test/distrib/python/test_packages.sh b/test/distrib/python/test_packages.sh index e16eddd5779..6bf49d45b99 100755 --- a/test/distrib/python/test_packages.sh +++ b/test/distrib/python/test_packages.sh @@ -22,12 +22,14 @@ shopt -s nullglob if [[ "$1" == "binary" ]] then echo "Testing Python binary distribution" - ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-*.whl) - TOOLS_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio_tools-*.whl) + ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-[0-9]*.whl) + TOOLS_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio_tools-[0-9]*.whl) else echo "Testing Python source distribution" - ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-*.tar.gz) - TOOLS_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio_tools-*.tar.gz) + ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-[0-9]*.tar.gz) + TOOLS_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-tools-[0-9]*.tar.gz) + HEALTH_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-health-checking-[0-9]*.tar.gz) + REFLECTION_ARCHIVES=("$EXTERNAL_GIT_ROOT"/input_artifacts/grpcio-reflection-[0-9]*.tar.gz) fi VIRTUAL_ENV=$(mktemp -d) @@ -52,6 +54,14 @@ function at_least_one_installs() { at_least_one_installs "${ARCHIVES[@]}" at_least_one_installs "${TOOLS_ARCHIVES[@]}" +if [[ "$1" == "source" ]] +then + echo "Testing Python health and reflection packages" + at_least_one_installs "${HEALTH_ARCHIVES[@]}" + at_least_one_installs "${REFLECTION_ARCHIVES[@]}" +fi + + # # Test our distributions # diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index bfb0a9e57fd..80adc201841 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -167,15 +167,15 @@ class PythonDistribTest(object): if self.source: return create_docker_jobspec( self.name, - 'tools/dockerfile/distribtest/python_dev_%s_%s' % ( - self.docker_suffix, self.arch), + 'tools/dockerfile/distribtest/python_dev_%s_%s' % + (self.docker_suffix, self.arch), 'test/distrib/python/run_source_distrib_test.sh', copy_rel_path='test/distrib') else: return create_docker_jobspec( self.name, - 'tools/dockerfile/distribtest/python_%s_%s' % ( - self.docker_suffix, self.arch), + 'tools/dockerfile/distribtest/python_%s_%s' % + (self.docker_suffix, self.arch), 'test/distrib/python/run_binary_distrib_test.sh', copy_rel_path='test/distrib') From 700d050500defd84161ddb32a883759a5b205d60 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Fri, 27 Apr 2018 07:01:00 -0700 Subject: [PATCH 4/4] Bump pip in yapf_virtual_environment --- tools/distrib/yapf_code.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh index 919e9c13d54..27c5e3129dd 100755 --- a/tools/distrib/yapf_code.sh +++ b/tools/distrib/yapf_code.sh @@ -30,11 +30,11 @@ EXCLUSIONS=( VIRTUALENV=yapf_virtual_environment -virtualenv $VIRTUALENV -PYTHON=$(realpath "${VIRTUALENV}/bin/python") -$PYTHON -m pip install --upgrade pip==9.0.2 -$PYTHON -m pip install --upgrade futures -$PYTHON -m pip install yapf==0.20.0 +python -m virtualenv $VIRTUALENV +PYTHON=${VIRTUALENV}/bin/python +"$PYTHON" -m pip install --upgrade pip==10.0.1 +"$PYTHON" -m pip install --upgrade futures +"$PYTHON" -m pip install yapf==0.20.0 yapf() { local exclusion exclusion_args=()