From 95e1ab6267a8a1d605cbcf17846a00b22419c8fc Mon Sep 17 00:00:00 2001 From: Sourabh Singh Date: Wed, 25 Sep 2024 08:47:32 -0700 Subject: [PATCH] unit testing setup and unit-tests script (#37070) 1. Added script to install all dependencies then run all unit tests for python grpcio. Closes #37070 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37070 from sourabhsinghs:feature/developer-setup-script 954892f20a8852ddb02090ef77e095359c089fea PiperOrigin-RevId: 678718224 --- tools/distrib/install_all_python_modules.sh | 73 ++++++++++--------- .../install_python_modules_and_run_tests.sh | 25 +++++++ tools/distrib/python_tooling_tests.sh | 28 +++++++ .../linux/grpc_toolingtests_python.sh | 26 +++++++ .../pull_request/grpc_toolingtests_python.cfg | 26 +++++++ 5 files changed, 144 insertions(+), 34 deletions(-) create mode 100755 tools/distrib/install_python_modules_and_run_tests.sh create mode 100755 tools/distrib/python_tooling_tests.sh create mode 100755 tools/internal_ci/linux/grpc_toolingtests_python.sh create mode 100644 tools/internal_ci/linux/pull_request/grpc_toolingtests_python.cfg diff --git a/tools/distrib/install_all_python_modules.sh b/tools/distrib/install_all_python_modules.sh index b389df48377..b23b38cc624 100755 --- a/tools/distrib/install_all_python_modules.sh +++ b/tools/distrib/install_all_python_modules.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 The gRPC Authors +# Copyright 2024 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,45 +13,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: Integrate this into CI to avoid bitrot. +set -ex -echo "It's recommended that you run this script from a virtual environment." +BASEDIR=$(dirname "$0")/../.. +PACKAGES="grpcio_channelz grpcio_csds grpcio_admin grpcio_health_checking grpcio_reflection grpcio_status grpcio_testing grpcio_csm_observability grpcio_tests" + +# Change to grpc repo root +cd "$BASEDIR"; +# unit-tests setup starts from here function maybe_run_command () { - if python setup.py --help-commands | grep "$1" &>/dev/null; then - python setup.py "$1"; + if python3 setup.py --help-commands | grep "$1" &>/dev/null; then + python3 setup.py "$1"; fi } -set -e +python3 -m pip install --upgrade "cython<4.0.0rc1"; +python3 setup.py install; -BASEDIR=$(dirname "$0") -BASEDIR=$(realpath "$BASEDIR")/../.. +# Build and install grpcio_tools +pushd tools/distrib/python/grpcio_tools; + ../make_grpcio_tools.py + GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install . +popd; -PACKAGES="grpcio_channelz grpcio_csds grpcio_admin grpcio_health_checking grpcio_reflection grpcio_status grpcio_testing grpcio_csm_observability grpcio_tests" +# Build and install grpcio_observability +pushd src/python/grpcio_observability; + ./make_grpcio_observability.py + GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install . +popd; + +# Install xds_protos +pushd tools/distrib/python/xds_protos; + GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install . +popd; + +# Build and install individual gRPC packages +pushd src/python; + for PACKAGE in ${PACKAGES}; do + pushd "${PACKAGE}"; + python3 setup.py clean; + maybe_run_command preprocess + maybe_run_command build_package_protos + python3 -m pip install .; + popd; + done +popd; -(cd "$BASEDIR"; - pip install --upgrade "cython<3.0.0rc1"; - python setup.py install; - pushd tools/distrib/python/grpcio_tools; - ../make_grpcio_tools.py - GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install . - popd; - pushd src/python/grpcio_observability; - ./make_grpcio_observability.py - GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install . - popd; - pushd tools/distrib/python/xds_protos; - GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install . - popd; - pushd src/python; - for PACKAGE in ${PACKAGES}; do - pushd "${PACKAGE}"; - python setup.py clean; - maybe_run_command preprocess - maybe_run_command build_package_protos - python -m pip install .; - popd; - done - popd; -) diff --git a/tools/distrib/install_python_modules_and_run_tests.sh b/tools/distrib/install_python_modules_and_run_tests.sh new file mode 100755 index 00000000000..d15c809db95 --- /dev/null +++ b/tools/distrib/install_python_modules_and_run_tests.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2024 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +BASEDIR=$(dirname "$0")/../.. +cd "$BASEDIR"; + +# install python all modules +./tools/distrib/install_all_python_modules.sh + +# run python tooling tests +./tools/distrib/python_tooling_tests.sh \ No newline at end of file diff --git a/tools/distrib/python_tooling_tests.sh b/tools/distrib/python_tooling_tests.sh new file mode 100755 index 00000000000..69007844241 --- /dev/null +++ b/tools/distrib/python_tooling_tests.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2024 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +BASEDIR=$(dirname "$0")/../.. +cd "$BASEDIR"; + +# Run tests for grpcio_tests +pushd src/python/grpcio_tests; + python3 setup.py test_lite + python3 setup.py test_aio + python3 setup.py test_py3_only +popd; + +chmod -R 755 src/ diff --git a/tools/internal_ci/linux/grpc_toolingtests_python.sh b/tools/internal_ci/linux/grpc_toolingtests_python.sh new file mode 100755 index 00000000000..13a7bd5a211 --- /dev/null +++ b/tools/internal_ci/linux/grpc_toolingtests_python.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright 2024 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +export DOCKERFILE_DIR=tools/dockerfile/distribtest/python_dev_ubuntu2204_x64 +export DOCKER_RUN_SCRIPT=tools/distrib/install_python_modules_and_run_tests.sh +export GRPC_TEST_REPORT_BASE_DIR=reports + +exec tools/run_tests/dockerize/build_and_run_docker.sh diff --git a/tools/internal_ci/linux/pull_request/grpc_toolingtests_python.cfg b/tools/internal_ci/linux/pull_request/grpc_toolingtests_python.cfg new file mode 100644 index 00000000000..41458d81748 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_toolingtests_python.cfg @@ -0,0 +1,26 @@ +# Copyright 2024 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_toolingtests_python.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +}