From 72888886d8304b05bc96655df6931ee11f2a2e65 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 22 Apr 2022 19:04:40 +0200 Subject: [PATCH] Cleanup run_in_docker.sh and corresponding examples. (#29469) * update docker_runners examples * run_in_docker improvements * run_in_docker.sh should use build_and_run_docker.sh * cleanup run_in_docker.sh and examples --- .../examples/bazel_test_in_docker.sh | 6 +- .../examples/coredump_in_docker.sh | 16 +++-- .../docker_runners/examples/gdb_in_docker.sh | 17 +++-- ..._in_docker.sh => run_tests_c_in_docker.sh} | 12 ++-- .../examples/run_tests_csharp_in_docker.sh | 2 +- tools/docker_runners/run_in_docker.sh | 72 ++++--------------- 6 files changed, 49 insertions(+), 76 deletions(-) rename tools/docker_runners/examples/{run_tests_c_cpp_in_docker.sh => run_tests_c_in_docker.sh} (65%) diff --git a/tools/docker_runners/examples/bazel_test_in_docker.sh b/tools/docker_runners/examples/bazel_test_in_docker.sh index 14d5ca35f72..ec673d7496a 100755 --- a/tools/docker_runners/examples/bazel_test_in_docker.sh +++ b/tools/docker_runners/examples/bazel_test_in_docker.sh @@ -21,9 +21,11 @@ cd "$(dirname "$0")/../../.." # TODO(jtattermusch): make sure bazel cache is persisted between runs # Note that the port server must be running so that the bazel tests can pass. +# (Run "tools/run_tests/start_port_server.py" first) # use the default docker image used for bazel builds export DOCKERFILE_DIR=tools/dockerfile/test/bazel -# TODO(jtattermusch): interestingly, the bazel build fails when "--privileged" docker arg is used (it probably has to do with sandboxing) -export DOCKER_EXTRA_ARGS="--privileged=false" +# Using host network allows using port server running on the host machine (and not just in the docker container) +# TODO(jtattermusch): interestingly, the bazel build fails when "--privileged=true" docker arg is used (it probably has to do with sandboxing) +export DOCKER_EXTRA_ARGS="--network=host" tools/docker_runners/run_in_docker.sh bazel test //test/... diff --git a/tools/docker_runners/examples/coredump_in_docker.sh b/tools/docker_runners/examples/coredump_in_docker.sh index 28b82f60d5d..e8352365e21 100755 --- a/tools/docker_runners/examples/coredump_in_docker.sh +++ b/tools/docker_runners/examples/coredump_in_docker.sh @@ -18,13 +18,17 @@ set -ex # change to grpc repo root cd "$(dirname "$0")/../../.." -# use the docker image used as the default for C++ by run_tests.py -# TODO(jtattermusch): document how to get the right docker image name -# for given run_tests.py --compiler/--arch params. -export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian9_x64 +# Use the docker image used as the default for C++ by run_tests.py +# To use the correct docker image for your experiments, +# note that every invocation of run_tests.py with "--use_docker" +# prints the docker image used as a debug message at the end of the run. +# This is expecially important when --compiler/--arch params are +# use, since they usually influence with docker image will be used +# by run_tests.py +export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian11_x64 -# add extra docker args if needed -export DOCKER_EXTRA_ARGS="" +# "--privileged" docker arg is required to be able to update /proc/sys/kernel/core_pattern +export DOCKER_EXTRA_ARGS="--privileged" # start the docker container with interactive shell tools/docker_runners/run_in_docker.sh bash diff --git a/tools/docker_runners/examples/gdb_in_docker.sh b/tools/docker_runners/examples/gdb_in_docker.sh index 6db4d9d14d4..2282dbdf614 100755 --- a/tools/docker_runners/examples/gdb_in_docker.sh +++ b/tools/docker_runners/examples/gdb_in_docker.sh @@ -18,13 +18,18 @@ set -ex # change to grpc repo root cd "$(dirname "$0")/../../.." -# use the docker image used as the default for C++ by run_tests.py -# TODO(jtattermusch): document how to get the right docker image name -# for given run_tests.py --compiler/--arch params. -export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian9_x64 +# Use the docker image used as the default for C++ by run_tests.py +# To use the correct docker image for your experiments, +# note that every invocation of run_tests.py with "--use_docker" +# prints the docker image used as a debug message at the end of the run. +# This is expecially important when --compiler/--arch params are +# use, since they usually influence with docker image will be used +# by run_tests.py +export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian11_x64 -# add extra docker args if needed -export DOCKER_EXTRA_ARGS="" +# "--privileged" docker arg is required to be disable address randomization by gdb +# TODO: is "--security-opt=seccomp=unconfined" actually needed? +export DOCKER_EXTRA_ARGS="--privileged --security-opt=seccomp=unconfined" # start the docker container with interactive shell tools/docker_runners/run_in_docker.sh bash diff --git a/tools/docker_runners/examples/run_tests_c_cpp_in_docker.sh b/tools/docker_runners/examples/run_tests_c_in_docker.sh similarity index 65% rename from tools/docker_runners/examples/run_tests_c_cpp_in_docker.sh rename to tools/docker_runners/examples/run_tests_c_in_docker.sh index 466b5a0b01f..fad762da4f6 100755 --- a/tools/docker_runners/examples/run_tests_c_cpp_in_docker.sh +++ b/tools/docker_runners/examples/run_tests_c_in_docker.sh @@ -19,7 +19,11 @@ set -ex cd "$(dirname "$0")/../../.." # use the docker image used as the default for C++ by run_tests.py -# TODO(jtattermusch): document how to get the right docker image name -# for given run_tests.py --compiler/--arch params. -export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian9_x64 -tools/docker_runners/run_in_docker.sh tools/run_tests/run_tests.py -l c c++ -c dbg +# To use the correct docker image for your experiments, +# note that every invocation of run_tests.py with "--use_docker" +# prints the docker image used as a debug message at the end of the run. +# This is expecially important when --compiler/--arch params are +# use, since they usually influence with docker image will be used +# by run_tests.py +export DOCKERFILE_DIR=tools/dockerfile/test/cxx_debian11_x64 +tools/docker_runners/run_in_docker.sh tools/run_tests/run_tests.py -l c -c dbg diff --git a/tools/docker_runners/examples/run_tests_csharp_in_docker.sh b/tools/docker_runners/examples/run_tests_csharp_in_docker.sh index 7252f7d2386..9192540bd6f 100755 --- a/tools/docker_runners/examples/run_tests_csharp_in_docker.sh +++ b/tools/docker_runners/examples/run_tests_csharp_in_docker.sh @@ -19,5 +19,5 @@ set -ex cd "$(dirname "$0")/../../.." # use the docker image used as the default for C# by run_tests.py -export DOCKERFILE_DIR=tools/dockerfile/test/csharp_buster_x64 +export DOCKERFILE_DIR=tools/dockerfile/test/csharp_debian11_x64 tools/docker_runners/run_in_docker.sh tools/run_tests/run_tests.py -l csharp -c dbg --compiler coreclr diff --git a/tools/docker_runners/run_in_docker.sh b/tools/docker_runners/run_in_docker.sh index 9c34e5e6ae6..be3340ec226 100755 --- a/tools/docker_runners/run_in_docker.sh +++ b/tools/docker_runners/run_in_docker.sh @@ -13,9 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Runs C# build in a docker container, but using the local workspace. -# Example usage: -# src/csharp/run_in_docker.sh tools/run_tests/run_tests.py -l csharp +# See tools/docker_runners/examples for more usage info. set -e @@ -36,60 +34,20 @@ then exit 1 fi -# Use image name based on Dockerfile location checksum -# For simplicity, currently only testing docker images that have already been pushed -# to dockerhub are supported (see tools/dockerfile/push_testing_images.sh) -# TODO(jtattermusch): add support for building dockerimages locally. -DOCKER_IMAGE=grpctesting/$(basename "$DOCKERFILE_DIR"):$(sha1sum "$DOCKERFILE_DIR/Dockerfile" | cut -f1 -d\ ) - -# TODO: support building dockerimage locally / pulling it from dockerhub - -# If TTY is available, the running container can be conveniently terminated with Ctrl+C. -if [[ -t 0 ]]; then - DOCKER_TTY_ARGS=("-it") -else - # The input device on kokoro is not a TTY, so -it does not work. - DOCKER_TTY_ARGS=() -fi - -# args required to be able to run gdb/strace under the docker container -DOCKER_PRIVILEGED_ARGS=( - "--privileged" - "--cap-add=SYS_PTRACE" - "--security-opt=seccomp=unconfined" -) - -DOCKER_NETWORK_ARGS=( - # enable IPv6 - "--sysctl=net.ipv6.conf.all.disable_ipv6=0" - # use host network, required for the port server to work correctly - "--network=host" +DOCKER_NONROOT_ARGS=( + # run under current user's UID and GID + # Uncomment to run the docker container as current user's UID and GID. + # That way, the files written by the container won't be owned by root (=you won't end up with polluted workspace), + # but it can have some other disadvantages. E.g.: + # - you won't be able install stuff inside the container + # - the home directory inside the container will be broken (you won't be able to write in it). + # That may actually break some language runtimes completely (e.g. grpc python might not build) + # "--user=$(id -u):$(id -g)" ) -DOCKER_CLEANUP_ARGS=( - # delete the container when the containers exits - # (otherwise the container will not release the disk space it used) - "--rm=true" -) - -DOCKER_PROPAGATE_ENV_ARGS=( - "--env-file=tools/run_tests/dockerize/docker_propagate_env.list" \ -) - -# Uncomment to run the docker container as current user's UID and GID. -# That way, the files written by the container won't be owned by root (=you won't end up with polluted workspace), -# but it can have some other disadvantages. E.g.: -# - you won't be able install stuff inside the container -# - the home directory inside the container will be broken (you won't be able to write in it). -# That may actually break some language runtimes completely (e.g. grpc python might not build) -# DOCKER_NONROOT_ARGS=( -# # run under current user's UID and GID -# "--user=$(id -u):$(id -g)" -# ) - -# Enable command echo just before running the final docker command to make the docker args visible. -set -ex +# the original DOCKER_EXTRA_ARGS + all the args defined in this script +export DOCKER_EXTRA_ARGS="${DOCKER_NONROOT_ARGS[@]} ${DOCKER_EXTRA_ARGS}" +# download the docker images from dockerhub instead of building them locally +export DOCKERHUB_ORGANIZATION=grpctesting -# Run command inside C# docker container. -# - the local clone of grpc repository will be mounted as /workspace. -exec docker run "${DOCKER_TTY_ARGS[@]}" "${DOCKER_PRIVILEGED_ARGS[@]}" "${DOCKER_NETWORK_ARGS[@]}" "${DOCKER_CLEANUP_ARGS[@]}" "${DOCKER_PROPAGATE_ENV_ARGS[@]}" ${DOCKER_EXTRA_ARGS} -v "${grpc_rootdir}":/workspace -w /workspace "${DOCKER_IMAGE}" bash -c "$*" +exec tools/run_tests/dockerize/build_and_run_docker.sh "$@"