Fix Race Condition During grpcio-tests Install on CI (#30093)

* Tryonly running fetch_build_eggs once

* Run pip freeze

* Use the correct pip

* Try explicitly printing stacktrace

* Try to pin/upgrade the wheel package

* Try running only 3.8

* Show the initial state of the egg directory

* Whoops

* More permissive

* What does this button do?

* Try just 3.9

* Try 3.10

* Back to 3.8

* Try pre-installing protobuf

* Clean up

* Clean up better

tools/run_tests/helper_scripts/build_python.sh

@@ -168,6 +168,14 @@ pip_install_dir() {
   cd "$PWD"
 }
 
+pip_install_dir_and_deps() {
+  PWD=$(pwd)
+  cd "$1"
+  ($VENV_PYTHON setup.py build_ext -c "$TOOLCHAIN" || true)
+  $VENV_PYTHON -m pip install .
+  cd "$PWD"
+}
+
 # Install gevent
 if [[ "$VENV" == "py36" ]]; then
   # TODO(https://github.com/grpc/grpc/issues/15411) unpin this
@@ -187,7 +195,7 @@ fi
 pip_install_dir "$ROOT"
 
 $VENV_PYTHON "$ROOT/tools/distrib/python/make_grpcio_tools.py"
-pip_install_dir "$ROOT/tools/distrib/python/grpcio_tools"
+pip_install_dir_and_deps "$ROOT/tools/distrib/python/grpcio_tools"
 
 # Build/install Channelz
 $VENV_PYTHON "$ROOT/src/python/grpcio_channelz/setup.py" preprocess