unit testing setup and unit-tests script (#37070)

1. Added script to install all dependencies then run all unit tests for python grpcio.

Closes #37070

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37070 from sourabhsinghs:feature/developer-setup-script 954892f20a
PiperOrigin-RevId: 678718224

tools/distrib/install_all_python_modules.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2020 The gRPC Authors
+# Copyright 2024 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,45 +13,50 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: Integrate this into CI to avoid bitrot.
+set -ex
 
-echo "It's recommended that you run this script from a virtual environment."
+BASEDIR=$(dirname "$0")/../..
+PACKAGES="grpcio_channelz  grpcio_csds  grpcio_admin grpcio_health_checking  grpcio_reflection  grpcio_status  grpcio_testing grpcio_csm_observability grpcio_tests"
+
+# Change to grpc repo root
+cd "$BASEDIR";
 
+# unit-tests setup starts from here
 function maybe_run_command () {
-  if python setup.py --help-commands | grep "$1" &>/dev/null; then
-    python setup.py "$1";
+  if python3 setup.py --help-commands | grep "$1" &>/dev/null; then
+    python3 setup.py "$1";
   fi
 }
 
-set -e
+python3 -m pip install --upgrade "cython<4.0.0rc1";
+python3 setup.py install;
 
-BASEDIR=$(dirname "$0")
-BASEDIR=$(realpath "$BASEDIR")/../..
+# Build and install grpcio_tools
+pushd tools/distrib/python/grpcio_tools;
+  ../make_grpcio_tools.py
+  GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
+popd;
 
-PACKAGES="grpcio_channelz  grpcio_csds  grpcio_admin grpcio_health_checking  grpcio_reflection  grpcio_status  grpcio_testing grpcio_csm_observability grpcio_tests"
+# Build and install grpcio_observability
+pushd src/python/grpcio_observability;
+  ./make_grpcio_observability.py
+  GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
+popd;
+
+# Install xds_protos
+pushd tools/distrib/python/xds_protos;
+  GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
+popd;
+
+# Build and install individual gRPC packages
+pushd src/python;
+  for PACKAGE in ${PACKAGES}; do
+    pushd "${PACKAGE}";
+      python3 setup.py clean;
+      maybe_run_command preprocess
+      maybe_run_command build_package_protos
+      python3 -m pip install .;
+    popd;
+  done
+popd;
 
-(cd "$BASEDIR";
-  pip install --upgrade "cython<3.0.0rc1";
-  python setup.py install;
-  pushd tools/distrib/python/grpcio_tools;
-    ../make_grpcio_tools.py
-    GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
-  popd;
-  pushd src/python/grpcio_observability;
-    ./make_grpcio_observability.py
-    GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
-  popd;
-  pushd tools/distrib/python/xds_protos;
-    GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .
-  popd;
-  pushd src/python;
-    for PACKAGE in ${PACKAGES}; do
-      pushd "${PACKAGE}";
-        python setup.py clean;
-        maybe_run_command preprocess
-        maybe_run_command build_package_protos
-        python -m pip install .;
-      popd;
-    done
-  popd;
-)

tools/distrib/install_python_modules_and_run_tests.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright 2024 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+BASEDIR=$(dirname "$0")/../..
+cd "$BASEDIR";
+
+# install python all modules
+./tools/distrib/install_all_python_modules.sh
+
+# run python tooling tests
+./tools/distrib/python_tooling_tests.sh

tools/distrib/python_tooling_tests.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright 2024 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+BASEDIR=$(dirname "$0")/../..
+cd "$BASEDIR";
+
+# Run tests for grpcio_tests
+pushd src/python/grpcio_tests;
+  python3 setup.py test_lite
+  python3 setup.py test_aio
+  python3 setup.py test_py3_only
+popd;
+
+chmod -R 755 src/

tools/internal_ci/linux/grpc_toolingtests_python.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright 2024 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+# Enter the gRPC repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+export DOCKERFILE_DIR=tools/dockerfile/distribtest/python_dev_ubuntu2204_x64
+export DOCKER_RUN_SCRIPT=tools/distrib/install_python_modules_and_run_tests.sh
+export GRPC_TEST_REPORT_BASE_DIR=reports
+
+exec tools/run_tests/dockerize/build_and_run_docker.sh

tools/internal_ci/linux/pull_request/grpc_toolingtests_python.cfg

@@ -0,0 +1,26 @@
+# Copyright 2024 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_toolingtests_python.sh"
+timeout_mins: 240
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}