From 7069770ee0068dcde5ea86426979c71d5a47f09d Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 14 Jan 2022 09:25:33 -0800 Subject: [PATCH] Merge the 3 repeating Python binary compilations (Attempt 2) (#28543) * Revert "Revert "Merge the 3 repeating Python binary compilations (#28500)" (#28539)" This reverts commit 0554cbee9cfcb58c3fe7c55300f6f447998c1963. * Update hardcoded Python path * Repect existing images and only use new path for newer releases * fix interop_matrix testcases Co-authored-by: Jan Tattermusch --- .gitignore | 23 ++- tools/distrib/docgen/_generate_python_doc.sh | 2 +- tools/interop_matrix/client_matrix.py | 8 +- tools/interop_matrix/testcases/python__master | 36 ++--- .../interop_matrix/testcases/python__v1.41.1 | 22 +++ .../run_tests/helper_scripts/build_python.sh | 11 +- tools/run_tests/helper_scripts/run_python.sh | 3 +- tools/run_tests/run_interop_tests.py | 4 +- tools/run_tests/run_tests.py | 140 +++++++++--------- tools/run_tests/run_tests_matrix.py | 2 +- 10 files changed, 132 insertions(+), 119 deletions(-) create mode 100755 tools/interop_matrix/testcases/python__v1.41.1 diff --git a/.gitignore b/.gitignore index d0d363e1a06..f236d45210b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,24 +6,23 @@ libs objs # Python items -cython_debug/ -python_build/ -yapf_virtual_environment/ -python_pylint_venv/ .coverage* .eggs -htmlcov/ -dist/ +.pytype *.egg -py27_gevent/ -py27_native/ -py3[0-9]_gevent/ -py3[0-9]_native/ +*.egg-info a.out +cython_debug/ +dist/ +htmlcov/ +py3*/ +python_build/ +python_pylint_venv/ +src/python/grpcio_*/=* +src/python/grpcio_*/build/ src/python/grpcio_*/LICENSE src/python/grpcio_status/grpc_status/google/rpc/status.proto -.pytype -*.egg-info +yapf_virtual_environment/ # Node installation output node_modules diff --git a/tools/distrib/docgen/_generate_python_doc.sh b/tools/distrib/docgen/_generate_python_doc.sh index febc3188aed..7f9c6c9d672 100755 --- a/tools/distrib/docgen/_generate_python_doc.sh +++ b/tools/distrib/docgen/_generate_python_doc.sh @@ -26,6 +26,6 @@ export HOME pip install -r requirements.bazel.txt tools/run_tests/run_tests.py -c opt -l python --compiler python3.8 --newline_on_success -j 8 --build_only # shellcheck disable=SC1091 -source py38_native/bin/activate +source py38/bin/activate pip install --upgrade Sphinx python setup.py doc diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index ff94880e716..d2a47ef91fe 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -391,8 +391,12 @@ LANG_RELEASE_MATRIX = { ('v1.39.0', ReleaseInfo(runtimes=['python'], testcases_file='python__v1.18.0')), - ('v1.41.1', ReleaseInfo(runtimes=['python'])), - ('v1.42.0', ReleaseInfo(runtimes=['python'])), + ('v1.41.1', + ReleaseInfo(runtimes=['python'], + testcases_file='python__v1.41.1')), + ('v1.42.0', + ReleaseInfo(runtimes=['python'], + testcases_file='python__v1.41.1')), ]), 'node': OrderedDict([ diff --git a/tools/interop_matrix/testcases/python__master b/tools/interop_matrix/testcases/python__master index 7a07133bbff..0b9ea8d1a95 100755 --- a/tools/interop_matrix/testcases/python__master +++ b/tools/interop_matrix/testcases/python__master @@ -2,21 +2,21 @@ # DO NOT MODIFY # This file is generated by run_interop_tests.py/create_testcases.sh echo "Testing ${docker_image:=grpc_interop_python:0de8fc68-43f4-4fa0-8738-1228de6ffe8c}" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=large_unary --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=empty_unary --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=ping_pong --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=empty_stream --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=client_streaming --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=server_streaming --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_begin --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_first_response --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=timeout_on_sleeping_server --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=large_unary --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=empty_unary --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=ping_pong --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=empty_stream --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=client_streaming --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=server_streaming --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_begin --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_first_response --use_tls=true\"" -docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=timeout_on_sleeping_server --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=large_unary --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=empty_unary --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=ping_pong --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=empty_stream --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=client_streaming --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=server_streaming --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_begin --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_first_response --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=timeout_on_sleeping_server --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=large_unary --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=empty_unary --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=ping_pong --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=empty_stream --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=client_streaming --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=server_streaming --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_begin --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_first_response --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=timeout_on_sleeping_server --use_tls=true\"" diff --git a/tools/interop_matrix/testcases/python__v1.41.1 b/tools/interop_matrix/testcases/python__v1.41.1 new file mode 100755 index 00000000000..7a07133bbff --- /dev/null +++ b/tools/interop_matrix/testcases/python__v1.41.1 @@ -0,0 +1,22 @@ +#!/bin/bash +# DO NOT MODIFY +# This file is generated by run_interop_tests.py/create_testcases.sh +echo "Testing ${docker_image:=grpc_interop_python:0de8fc68-43f4-4fa0-8738-1228de6ffe8c}" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=large_unary --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=empty_unary --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=ping_pong --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=empty_stream --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=client_streaming --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=server_streaming --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_begin --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_first_response --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test.sandbox.googleapis.com --server_port=443 --test_case=timeout_on_sleeping_server --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=large_unary --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=empty_unary --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=ping_pong --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=empty_stream --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=client_streaming --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=server_streaming --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_begin --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=cancel_after_first_response --use_tls=true\"" +docker run -i --rm=true -e PYTHONPATH=/var/local/git/grpc/src/python/gens -e LD_LIBRARY_PATH=/var/local/git/grpc/libs/opt -w /var/local/git/grpc --net=host $docker_image bash -c "py39_native/bin/python src/python/grpcio_tests/setup.py run_interop --client --args=\"--server_host=grpc-test4.sandbox.googleapis.com --server_port=443 --test_case=timeout_on_sleeping_server --use_tls=true\"" diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 4dc1a3f3607..bc034cec105 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -176,16 +176,13 @@ pip_install_dir() { cd "$PWD" } -case "$VENV" in - *py36_gevent*) +# Install gevent +if [[ "$VENV" == "py36" ]]; then # TODO(https://github.com/grpc/grpc/issues/15411) unpin this pip_install gevent==1.3.b1 - ;; - *gevent*) +else pip_install -U gevent - ;; -esac - +fi pip_install --upgrade cython pip_install --upgrade six protobuf diff --git a/tools/run_tests/helper_scripts/run_python.sh b/tools/run_tests/helper_scripts/run_python.sh index 2b7321e527a..72e659125d8 100755 --- a/tools/run_tests/helper_scripts/run_python.sh +++ b/tools/run_tests/helper_scripts/run_python.sh @@ -18,7 +18,7 @@ set -ex # change to grpc repo root cd "$(dirname "$0")/../../.." -PYTHON=$(realpath "${1:-py27/bin/python}") +PYTHON=$(realpath "${1:-py36/bin/python}") ROOT=$(pwd) @@ -27,4 +27,3 @@ $PYTHON "$ROOT/src/python/grpcio_tests/setup.py" "$2" mkdir -p "$ROOT/reports" rm -rf "$ROOT/reports/python-coverage" (mv -T "$ROOT/htmlcov" "$ROOT/reports/python-coverage") || true - diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 995d8aa9f0e..446742434e2 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -596,7 +596,7 @@ class RubyLanguage: return 'ruby' -_PYTHON_BINARY = 'py39_native/bin/python' +_PYTHON_BINARY = 'py39/bin/python' class PythonLanguage: @@ -673,7 +673,7 @@ class PythonAsyncIOLanguage: def server_cmd(self, args): return [ _PYTHON_BINARY, 'src/python/grpcio_tests/setup.py', - 'py39_native/bin/python', 'src/python/grpcio_tests/setup.py', + 'py39/bin/python', 'src/python/grpcio_tests/setup.py', '--args="{}"'.format(' '.join(args)) ] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 3b063f3c34f..a9219c467eb 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -178,22 +178,18 @@ _PythonConfigVars = collections.namedtuple('_ConfigVars', [ 'venv_relative_python', 'toolchain', 'runner', - 'test_name', - 'iomgr_platform', ]) def _python_config_generator(name, major, minor, bits, config_vars): - name += '_' + config_vars.iomgr_platform - return PythonConfig( - name, config_vars.shell + config_vars.builder + - config_vars.builder_prefix_arguments + - [_python_pattern_function(major=major, minor=minor, bits=bits)] + - [name] + config_vars.venv_relative_python + config_vars.toolchain, - config_vars.shell + config_vars.runner + [ - os.path.join(name, config_vars.venv_relative_python[0]), - config_vars.test_name - ]) + build = (config_vars.shell + config_vars.builder + + config_vars.builder_prefix_arguments + + [_python_pattern_function(major=major, minor=minor, bits=bits)] + + [name] + config_vars.venv_relative_python + config_vars.toolchain) + run = (config_vars.shell + config_vars.runner + [ + os.path.join(name, config_vars.venv_relative_python[0]), + ]) + return PythonConfig(name, build, run) def _pypy_config_generator(name, major, config_vars): @@ -614,8 +610,9 @@ class PythonLanguage(object): ], 'asyncio': ['src/python/grpcio_tests/tests_aio/tests.json'], } - _TEST_FOLDER = { - 'native': 'test', + + _TEST_COMMAND = { + 'native': 'test_lite', 'gevent': 'test_gevent', 'asyncio': 'test_aio', } @@ -627,28 +624,36 @@ class PythonLanguage(object): def test_specs(self): # load list of known test suites - tests_json = [] - for tests_json_file_name in self._TEST_SPECS_FILE[ - self.args.iomgr_platform]: - with open(tests_json_file_name) as tests_json_file: - tests_json.extend(json.load(tests_json_file)) - environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) - # TODO(https://github.com/grpc/grpc/issues/21401) Fork handlers is not - # designed for non-native IO manager. It has a side-effect that - # overrides threading settings in C-Core. - if args.iomgr_platform != 'native': - environment['GRPC_ENABLE_FORK_SUPPORT'] = '0' - return [ - self.config.job_spec( - config.run, - timeout_seconds=8 * 60, - environ=dict(GRPC_PYTHON_TESTRUNNER_FILTER=str(suite_name), - **environment), - shortname='%s.%s.%s' % - (config.name, self._TEST_FOLDER[self.args.iomgr_platform], - suite_name), - ) for suite_name in tests_json for config in self.pythons - ] + jobs = [] + for io_platform in self._TEST_SPECS_FILE: + test_cases = [] + for tests_json_file_name in self._TEST_SPECS_FILE[io_platform]: + with open(tests_json_file_name) as tests_json_file: + test_cases.extend(json.load(tests_json_file)) + + environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) + # TODO(https://github.com/grpc/grpc/issues/21401) Fork handlers is not + # designed for non-native IO manager. It has a side-effect that + # overrides threading settings in C-Core. + if io_platform != 'native': + environment['GRPC_ENABLE_FORK_SUPPORT'] = '0' + for python_config in self.pythons: + # TODO(https://github.com/grpc/grpc/issues/23784) allow gevent + # to run on later version once issue solved. + if io_platform == 'gevent' and python_config.name != 'py36': + continue + jobs.extend([ + self.config.job_spec( + python_config.run + [self._TEST_COMMAND[io_platform]], + timeout_seconds=8 * 60, + environ=dict( + GRPC_PYTHON_TESTRUNNER_FILTER=str(test_case), + **environment), + shortname='%s.%s.%s' % + (python_config.name, io_platform, test_case), + ) for test_case in test_cases + ]) + return jobs def pre_build_steps(self): return [] @@ -686,6 +691,11 @@ class PythonLanguage(object): def _get_pythons(self, args): """Get python runtimes to test with, based on current platform, architecture, compiler etc.""" + if args.iomgr_platform != 'native': + raise ValueError( + 'Python builds no longer differentiate IO Manager platforms, please use "native"' + ) + if args.arch == 'x86': bits = '32' else: @@ -710,25 +720,13 @@ class PythonLanguage(object): venv_relative_python = ['bin/python'] toolchain = ['unix'] - # Selects the corresponding testing mode. - # See src/python/grpcio_tests/commands.py for implementation details. - if args.iomgr_platform == 'native': - test_command = 'test_lite' - elif args.iomgr_platform == 'gevent': - test_command = 'test_gevent' - elif args.iomgr_platform == 'asyncio': - test_command = 'test_aio' - else: - raise ValueError('Unsupported IO Manager platform: %s' % - args.iomgr_platform) runner = [ os.path.abspath('tools/run_tests/helper_scripts/run_python.sh') ] config_vars = _PythonConfigVars(shell, builder, builder_prefix_arguments, - venv_relative_python, toolchain, runner, - test_command, args.iomgr_platform) + venv_relative_python, toolchain, runner) python36_config = _python_config_generator(name='py36', major='3', minor='6', @@ -749,6 +747,11 @@ class PythonLanguage(object): minor='9', bits=bits, config_vars=config_vars) + python310_config = _python_config_generator(name='py310', + major='3', + minor='10', + bits=bits, + config_vars=config_vars) pypy27_config = _pypy_config_generator(name='pypy', major='2', config_vars=config_vars) @@ -756,34 +759,19 @@ class PythonLanguage(object): major='3', config_vars=config_vars) - if args.iomgr_platform in ('asyncio', 'gevent'): - if args.compiler not in ('default', 'python3.6', 'python3.7', - 'python3.8', 'python3.9'): - raise Exception( - 'Compiler %s not supported with IO Manager platform: %s' % - (args.compiler, args.iomgr_platform)) - if args.compiler == 'default': if os.name == 'nt': - if args.iomgr_platform == 'gevent': - # TODO(https://github.com/grpc/grpc/issues/23784) allow - # gevent to run on later version once issue solved. - return (python36_config,) - else: - return (python38_config,) + return (python38_config,) + elif os.uname()[0] == 'Darwin': + # NOTE(rbellevi): Testing takes significantly longer on + # MacOS, so we restrict the number of interpreter versions + # tested. + return (python38_config,) else: - if args.iomgr_platform in ('asyncio', 'gevent'): - return (python36_config, python38_config) - elif os.uname()[0] == 'Darwin': - # NOTE(rbellevi): Testing takes significantly longer on - # MacOS, so we restrict the number of interpreter versions - # tested. - return (python38_config,) - else: - return ( - python37_config, - python38_config, - ) + return ( + python36_config, + python38_config, + ) elif args.compiler == 'python3.6': return (python36_config,) elif args.compiler == 'python3.7': @@ -792,6 +780,8 @@ class PythonLanguage(object): return (python38_config,) elif args.compiler == 'python3.9': return (python39_config,) + elif args.compiler == 'python3.10': + return (python310_config,) elif args.compiler == 'pypy': return (pypy27_config,) elif args.compiler == 'pypy3': @@ -803,6 +793,8 @@ class PythonLanguage(object): python36_config, python37_config, python38_config, + python39_config, + python310_config, ) else: raise Exception('Compiler %s not supported.' % args.compiler) diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index ff8cf31c33f..1033afe71d0 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -223,7 +223,7 @@ def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS): test_jobs += _generate_jobs(languages=['python'], configs=['opt'], platforms=['linux', 'macos', 'windows'], - iomgr_platforms=['native', 'gevent', 'asyncio'], + iomgr_platforms=['native'], labels=['basictests', 'multilang'], extra_args=extra_args + ['--report_multi_target'],