Cleanup run_in_docker.sh and corresponding examples. (#29469)

* update docker_runners examples

* run_in_docker improvements

* run_in_docker.sh should use build_and_run_docker.sh

* cleanup run_in_docker.sh and examples

tools/docker_runners/examples/bazel_test_in_docker.sh

@@ -21,9 +21,11 @@ cd "$(dirname "$0")/../../.."
 # TODO(jtattermusch): make sure bazel cache is persisted between runs
 
 # Note that the port server must be running so that the bazel tests can pass.
+# (Run "tools/run_tests/start_port_server.py" first)
 
 # use the default docker image used for bazel builds
 export DOCKERFILE_DIR=tools/dockerfile/test/bazel
-# TODO(jtattermusch): interestingly, the bazel build fails when "--privileged" docker arg is used (it probably has to do with sandboxing)
-export DOCKER_EXTRA_ARGS="--privileged=false"
+# Using host network allows using port server running on the host machine (and not just in the docker container)
+# TODO(jtattermusch): interestingly, the bazel build fails when "--privileged=true" docker arg is used (it probably has to do with sandboxing)
+export DOCKER_EXTRA_ARGS="--network=host"
 tools/docker_runners/run_in_docker.sh bazel test //test/...

tools/docker_runners/examples/coredump_in_docker.sh

@@ -18,13 +18,17 @@ set -ex
 # change to grpc repo root
 cd "$(dirname "$0")/../../.."
 
-# use the docker image used as the default for C++ by run_tests.py
-# TODO(jtattermusch): document how to get the right docker image name
-# for given run_tests.py --compiler/--arch params.
-export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian9_x64
+# Use the docker image used as the default for C++ by run_tests.py
+# To use the correct docker image for your experiments,
+# note that every invocation of run_tests.py with "--use_docker"
+# prints the docker image used as a debug message at the end of the run.
+# This is expecially important when --compiler/--arch params are
+# use, since they usually influence with docker image will be used
+# by run_tests.py
+export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian11_x64
 
-# add extra docker args if needed
-export DOCKER_EXTRA_ARGS=""
+# "--privileged" docker arg is required to be able to update /proc/sys/kernel/core_pattern
+export DOCKER_EXTRA_ARGS="--privileged"
 
 # start the docker container with interactive shell
 tools/docker_runners/run_in_docker.sh bash

tools/docker_runners/examples/gdb_in_docker.sh

@@ -18,13 +18,18 @@ set -ex
 # change to grpc repo root
 cd "$(dirname "$0")/../../.."
 
-# use the docker image used as the default for C++ by run_tests.py
-# TODO(jtattermusch): document how to get the right docker image name
-# for given run_tests.py --compiler/--arch params.
-export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian9_x64
+# Use the docker image used as the default for C++ by run_tests.py
+# To use the correct docker image for your experiments,
+# note that every invocation of run_tests.py with "--use_docker"
+# prints the docker image used as a debug message at the end of the run.
+# This is expecially important when --compiler/--arch params are
+# use, since they usually influence with docker image will be used
+# by run_tests.py
+export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian11_x64
 
-# add extra docker args if needed
-export DOCKER_EXTRA_ARGS=""
+# "--privileged" docker arg is required to be disable address randomization by gdb
+# TODO: is "--security-opt=seccomp=unconfined" actually needed?
+export DOCKER_EXTRA_ARGS="--privileged --security-opt=seccomp=unconfined"
 
 # start the docker container with interactive shell
 tools/docker_runners/run_in_docker.sh bash

tools/docker_runners/examples/run_tests_c_in_docker.sh

@@ -19,7 +19,11 @@ set -ex
 cd "$(dirname "$0")/../../.."
 
 # use the docker image used as the default for C++ by run_tests.py
-# TODO(jtattermusch): document how to get the right docker image name
-# for given run_tests.py --compiler/--arch params.
-export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian9_x64
-tools/docker_runners/run_in_docker.sh tools/run_tests/run_tests.py -l c c++ -c dbg
+# To use the correct docker image for your experiments,
+# note that every invocation of run_tests.py with "--use_docker"
+# prints the docker image used as a debug message at the end of the run.
+# This is expecially important when --compiler/--arch params are
+# use, since they usually influence with docker image will be used
+# by run_tests.py
+export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian11_x64
+tools/docker_runners/run_in_docker.sh tools/run_tests/run_tests.py -l c -c dbg

tools/docker_runners/examples/run_tests_csharp_in_docker.sh

@@ -19,5 +19,5 @@ set -ex
 cd "$(dirname "$0")/../../.."
 
 # use the docker image used as the default for C# by run_tests.py
-export DOCKERFILE_DIR=tools/dockerfile/test/csharp_buster_x64
+export DOCKERFILE_DIR=tools/dockerfile/test/csharp_debian11_x64
 tools/docker_runners/run_in_docker.sh tools/run_tests/run_tests.py -l csharp -c dbg --compiler coreclr

tools/docker_runners/run_in_docker.sh

@@ -13,9 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Runs C# build in a docker container, but using the local workspace.
-# Example usage:
-# src/csharp/run_in_docker.sh tools/run_tests/run_tests.py -l csharp
+# See tools/docker_runners/examples for more usage info.
 
 set -e
 
@@ -36,60 +34,20 @@ then
     exit 1
 fi
 
-# Use image name based on Dockerfile location checksum
-# For simplicity, currently only testing docker images that have already been pushed
-# to dockerhub are supported (see tools/dockerfile/push_testing_images.sh)
-# TODO(jtattermusch): add support for building dockerimages locally.
-DOCKER_IMAGE=grpctesting/$(basename "$DOCKERFILE_DIR"):$(sha1sum "$DOCKERFILE_DIR/Dockerfile" | cut -f1 -d\ )
-
-# TODO: support building dockerimage locally / pulling it from dockerhub
-
-# If TTY is available, the running container can be conveniently terminated with Ctrl+C.
-if [[ -t 0 ]]; then
-  DOCKER_TTY_ARGS=("-it")
-else
-  # The input device on kokoro is not a TTY, so -it does not work.
-  DOCKER_TTY_ARGS=()
-fi
-
-# args required to be able to run gdb/strace under the docker container
-DOCKER_PRIVILEGED_ARGS=(
-  "--privileged"
-  "--cap-add=SYS_PTRACE"
-  "--security-opt=seccomp=unconfined"
-)
-
-DOCKER_NETWORK_ARGS=(
-  # enable IPv6
-  "--sysctl=net.ipv6.conf.all.disable_ipv6=0"
-  # use host network, required for the port server to work correctly
-  "--network=host"
+DOCKER_NONROOT_ARGS=(
+  # run under current user's UID and GID
+  # Uncomment to run the docker container as current user's UID and GID.
+  # That way, the files written by the container won't be owned by root (=you won't end up with polluted workspace),
+  # but it can have some other disadvantages. E.g.:
+  # - you won't be able install stuff inside the container
+  # - the home directory inside the container will be broken (you won't be able to write in it).
+  #   That may actually break some language runtimes completely (e.g. grpc python might not build)
+  # "--user=$(id -u):$(id -g)"
 )
 
-DOCKER_CLEANUP_ARGS=(
-  # delete the container when the containers exits
-  # (otherwise the container will not release the disk space it used)
-  "--rm=true"
-)
-
-DOCKER_PROPAGATE_ENV_ARGS=(
-  "--env-file=tools/run_tests/dockerize/docker_propagate_env.list" \
-)
-
-# Uncomment to run the docker container as current user's UID and GID.
-# That way, the files written by the container won't be owned by root (=you won't end up with polluted workspace),
-# but it can have some other disadvantages. E.g.:
-# - you won't be able install stuff inside the container
-# - the home directory inside the container will be broken (you won't be able to write in it).
-#   That may actually break some language runtimes completely (e.g. grpc python might not build)
-# DOCKER_NONROOT_ARGS=(
-#   # run under current user's UID and GID
-#   "--user=$(id -u):$(id -g)"
-# )
-
-# Enable command echo just before running the final docker command to make the docker args visible.
-set -ex
+# the original DOCKER_EXTRA_ARGS + all the args defined in this script
+export DOCKER_EXTRA_ARGS="${DOCKER_NONROOT_ARGS[@]} ${DOCKER_EXTRA_ARGS}"
+# download the docker images from dockerhub instead of building them locally
+export DOCKERHUB_ORGANIZATION=grpctesting
 
-# Run command inside C# docker container.
-# - the local clone of grpc repository will be mounted as /workspace.
-exec docker run "${DOCKER_TTY_ARGS[@]}" "${DOCKER_PRIVILEGED_ARGS[@]}" "${DOCKER_NETWORK_ARGS[@]}" "${DOCKER_CLEANUP_ARGS[@]}" "${DOCKER_PROPAGATE_ENV_ARGS[@]}" ${DOCKER_EXTRA_ARGS} -v "${grpc_rootdir}":/workspace -w /workspace "${DOCKER_IMAGE}" bash -c "$*"
+exec tools/run_tests/dockerize/build_and_run_docker.sh "$@"