From 23c32621d6336fa2540d1872e607b80d7594e059 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 2 Apr 2020 15:36:07 -0700 Subject: [PATCH 1/8] Fix all_the_cpythons. Add 3.5 to defaults --- .../Dockerfile.template | 6 ++++- third_party/abseil-cpp | 2 +- .../python_stretch_default_x64/Dockerfile | 23 +++++++++++++++++++ tools/run_tests/run_tests.py | 1 + 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/templates/tools/dockerfile/test/python_stretch_default_x64/Dockerfile.template b/templates/tools/dockerfile/test/python_stretch_default_x64/Dockerfile.template index ccb88e75302..bb4a9f29c01 100644 --- a/templates/tools/dockerfile/test/python_stretch_default_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/python_stretch_default_x64/Dockerfile.template @@ -16,7 +16,11 @@ <%include file="../../python_stretch.include"/> <%include file="../../compile_python_36.include"/> - + <%include file="../../compile_python_38.include"/> + + RUN apt-get update && apt-get install -y python3.5 python3.5-dev + RUN curl https://bootstrap.pypa.io/get-pip.py | python3.5 + RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7 diff --git a/third_party/abseil-cpp b/third_party/abseil-cpp index df3ea785d8c..b832dce8489 160000 --- a/third_party/abseil-cpp +++ b/third_party/abseil-cpp @@ -1 +1 @@ -Subproject commit df3ea785d8c30a9503321a3d35ee7d35808f190d +Subproject commit b832dce8489ef7b6231384909fd9b68d5a5ff2b7 diff --git a/tools/dockerfile/test/python_stretch_default_x64/Dockerfile b/tools/dockerfile/test/python_stretch_default_x64/Dockerfile index 9a1d6c09deb..713fd6fe230 100644 --- a/tools/dockerfile/test/python_stretch_default_x64/Dockerfile +++ b/tools/dockerfile/test/python_stretch_default_x64/Dockerfile @@ -84,6 +84,29 @@ RUN cd /tmp && \ RUN python3.6 -m ensurepip && \ python3.6 -m pip install coverage +#================= +# Compile CPython 3.8.0b4 from source + +RUN apt-get update && apt-get install -y zlib1g-dev libssl-dev +RUN apt-get update && apt-get install -y jq build-essential libffi-dev + +RUN cd /tmp && \ + wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0b4.tgz && \ + tar xzvf Python-3.8.0b4.tgz && \ + cd Python-3.8.0b4 && \ + ./configure && \ + make install + +RUN cd /tmp && \ + echo "b8f4f897df967014ddb42033b90c3058 Python-3.8.0b4.tgz" > checksum.md5 && \ + md5sum -c checksum.md5 + +RUN python3.8 -m ensurepip && \ + python3.8 -m pip install coverage + + +RUN apt-get update && apt-get install -y python3.5 python3.5-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python3.5 RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7 diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index a3d255110f0..8ed4c88f683 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -869,6 +869,7 @@ class PythonLanguage(object): else: return ( python27_config, + python35_config, python36_config, python37_config, ) From 9344a259b5d1216a1ba2184dd494a35299969c30 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 6 Apr 2020 13:06:35 -0700 Subject: [PATCH 2/8] Fix MacOS Python 3.5 gevent build --- .../run_tests/helper_scripts/build_python.sh | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 670b5fc69d8..48b4d13a262 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -165,24 +165,33 @@ pip_install_dir() { cd "$PWD" } +# On library/version/platforms combos that do not have a binary +# published, we may end up building a dependency from source. In that +# case, several of our build environment variables may disrupt the +# third-party build process. This function pipes through only the +# minimal environment necessary. +pip_install() { + /usr/bin/env -i PATH="$PATH" $VENV_PYTHON -m pip install $@ +} + case "$VENV" in *py36_gevent*) # TODO(https://github.com/grpc/grpc/issues/15411) unpin this - $VENV_PYTHON -m pip install gevent==1.3.b1 + pip_install gevent==1.3.b1 ;; *gevent*) - $VENV_PYTHON -m pip install -U gevent + pip_install -U gevent ;; esac -$VENV_PYTHON -m pip install --upgrade pip==19.3.1 -$VENV_PYTHON -m pip install --upgrade setuptools -$VENV_PYTHON -m pip install --upgrade cython -$VENV_PYTHON -m pip install --upgrade six enum34 protobuf +pip_install --upgrade pip==19.3.1 +pip_install --upgrade setuptools +pip_install --upgrade cython +pip_install --upgrade six enum34 protobuf if [ "$("$VENV_PYTHON" -c "import sys; print(sys.version_info[0])")" == "2" ] then - $VENV_PYTHON -m pip install futures + pip_install futures fi pip_install_dir "$ROOT" @@ -214,7 +223,7 @@ pip_install_dir "$ROOT/src/python/grpcio_status" pip_install_dir "$ROOT/src/python/grpcio_testing" # Build/install tests -$VENV_PYTHON -m pip install coverage==4.4 oauth2client==4.1.0 \ +pip_install coverage==4.4 oauth2client==4.1.0 \ google-auth==1.0.0 requests==2.14.2 \ googleapis-common-protos==1.5.5 $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" preprocess From 4c79018a96a0e59de1a039f6f7af7c410509bdd8 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 6 Apr 2020 13:08:28 -0700 Subject: [PATCH 3/8] Revert changes to abseil-cpp --- third_party/abseil-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/abseil-cpp b/third_party/abseil-cpp index b832dce8489..df3ea785d8c 160000 --- a/third_party/abseil-cpp +++ b/third_party/abseil-cpp @@ -1 +1 @@ -Subproject commit b832dce8489ef7b6231384909fd9b68d5a5ff2b7 +Subproject commit df3ea785d8c30a9503321a3d35ee7d35808f190d From fb2b2122acdfcb854136fec9bf3916edb56ff102 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 6 Apr 2020 14:21:00 -0700 Subject: [PATCH 4/8] Add quotes --- tools/run_tests/helper_scripts/build_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 48b4d13a262..a633049510b 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -165,13 +165,13 @@ pip_install_dir() { cd "$PWD" } -# On library/version/platforms combos that do not have a binary +# On library/version/platforms combo that do not have a binary # published, we may end up building a dependency from source. In that # case, several of our build environment variables may disrupt the # third-party build process. This function pipes through only the # minimal environment necessary. pip_install() { - /usr/bin/env -i PATH="$PATH" $VENV_PYTHON -m pip install $@ + /usr/bin/env -i PATH="$PATH" "$VENV_PYTHON" -m pip install $@ } case "$VENV" in From 0aa77e410f90d5bf5b6111e79553a4ebff45594e Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 6 Apr 2020 16:40:06 -0700 Subject: [PATCH 5/8] Restrict set of interpreters on MacOS --- tools/run_tests/run_tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 8ed4c88f683..9008004cdad 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -866,6 +866,15 @@ class PythonLanguage(object): else: if args.iomgr_platform == 'asyncio': return (python36_config,) + else if os.uname()[0] == 'Darwin': + # NOTE(rbellevi): Testing takes significantly longer on + # MacOS, so we restrict the number of interpreter versions + # tested. + return ( + python27_config, + python36_config, + python37_config, + ) else: return ( python27_config, From 8079de2466a3077ff8facfd11cb6401d4b054dbb Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 7 Apr 2020 10:07:15 -0700 Subject: [PATCH 6/8] Accidentally a word --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 9008004cdad..b1c56762bd1 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -866,7 +866,7 @@ class PythonLanguage(object): else: if args.iomgr_platform == 'asyncio': return (python36_config,) - else if os.uname()[0] == 'Darwin': + elif os.uname()[0] == 'Darwin': # NOTE(rbellevi): Testing takes significantly longer on # MacOS, so we restrict the number of interpreter versions # tested. From dbf3bb31cdc3360c73f7c34c3d1c6a1261236db9 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 7 Apr 2020 11:09:27 -0700 Subject: [PATCH 7/8] Shellcheck --- tools/run_tests/helper_scripts/build_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index a633049510b..cfcb151bcce 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -171,7 +171,7 @@ pip_install_dir() { # third-party build process. This function pipes through only the # minimal environment necessary. pip_install() { - /usr/bin/env -i PATH="$PATH" "$VENV_PYTHON" -m pip install $@ + /usr/bin/env -i PATH="$PATH" "$VENV_PYTHON" -m pip install "$@" } case "$VENV" in From 1c1321ddea3fe5f2357872654d48ea4d1f1af54e Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 7 Apr 2020 16:10:20 -0700 Subject: [PATCH 8/8] Indentation --- tools/run_tests/helper_scripts/build_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index cfcb151bcce..e79a8896092 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -224,8 +224,8 @@ pip_install_dir "$ROOT/src/python/grpcio_testing" # Build/install tests pip_install coverage==4.4 oauth2client==4.1.0 \ - google-auth==1.0.0 requests==2.14.2 \ - googleapis-common-protos==1.5.5 + google-auth==1.0.0 requests==2.14.2 \ + googleapis-common-protos==1.5.5 $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" preprocess $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" build_package_protos pip_install_dir "$ROOT/src/python/grpcio_tests"