From 14f5d4768de9371c7160d896b3bcb0f8fe6d247f Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 6 Oct 2023 10:02:40 +0200 Subject: [PATCH] [bazelify tests] Add "concurrent_bazel.sh" wrapper to enable parallel invocations of bazel (#34568) (I had to first solve a problem with "dubious file ownership" error that was happening inside the grpc_repo_archive action when using the concurrent wrapper). Example: ``` tools/docker_runners/examples/concurrent_bazel.sh --bazelrc=tools/remote_build/linux.bazelrc test --genrule_strategy=remote,local --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh //tools/bazelify_tests/test:runtests_csharp_linux_dbg ``` --- ...d_dubious_ownership_error_allusers.include | 10 +++++ .../dockerfile/test/bazel/Dockerfile.template | 1 + .../dockerimage_current_versions.bzl | 2 +- .../examples/concurrent_bazel.sh | 42 +++++++++++++++++++ tools/dockerfile/test/bazel.current_version | 2 +- tools/dockerfile/test/bazel/Dockerfile | 11 +++++ 6 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 templates/tools/dockerfile/git_avoid_dubious_ownership_error_allusers.include create mode 100755 tools/docker_runners/examples/concurrent_bazel.sh diff --git a/templates/tools/dockerfile/git_avoid_dubious_ownership_error_allusers.include b/templates/tools/dockerfile/git_avoid_dubious_ownership_error_allusers.include new file mode 100644 index 00000000000..4365387cf05 --- /dev/null +++ b/templates/tools/dockerfile/git_avoid_dubious_ownership_error_allusers.include @@ -0,0 +1,10 @@ +#================= +# Setup git to access working directory across docker boundary. +# This avoids the "fatal: detected dubious ownership in repository XYZ" +# git error. Using "--system" makes the setting work +# for all users or even current user is not properly defined +# (which can happen e.g. inside execution environment +# of a bazel action) + +RUN git config --system --add safe.directory '*' +RUN git config --system protocol.file.allow always diff --git a/templates/tools/dockerfile/test/bazel/Dockerfile.template b/templates/tools/dockerfile/test/bazel/Dockerfile.template index 03028631581..6c00b5e0a90 100644 --- a/templates/tools/dockerfile/test/bazel/Dockerfile.template +++ b/templates/tools/dockerfile/test/bazel/Dockerfile.template @@ -17,6 +17,7 @@ <%include file="../../oss_fuzz_base.include"/> <%include file="../../bazel.include"/> <%include file="../../git_avoid_dubious_ownership_error.include"/> + <%include file="../../git_avoid_dubious_ownership_error_allusers.include"/> RUN apt-get install -y gdb diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl index 8cd4c5fc18e..ef80abfceba 100644 --- a/tools/bazelify_tests/dockerimage_current_versions.bzl +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -89,7 +89,7 @@ DOCKERIMAGE_CURRENT_VERSIONS = { "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:7b044d6848f82234dba81b38d8eca220b608f830f93b42932df59ed6fe20b24d", "tools/dockerfile/interoptest/lb_interop_fake_servers.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/lb_interop_fake_servers@sha256:b89a51dd9147e1293f50ee64dd719fce5929ca7894d3770a3d80dbdecb99fd52", "tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:64ffc5d1e117172ca4dda89720087616830996181192de25fe10e03a88f0b3e5", - "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:d3291d3129469bd4bdd58b5dd81eb6c5668548a4d6e6c8e952fb372d1874b078", + "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:023fdfd072e278d4129f2c8f8564f6697e767c9847aae8c9411ea853f3faaa6f", "tools/dockerfile/test/bazel_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64@sha256:1b7200c85ba8bc10529e1867a27ee81ad303ec8c8efec8bbc5863b4ee8ac6d03", "tools/dockerfile/test/binder_transport_apk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/binder_transport_apk@sha256:38c1da5544c717ef0e3d3492833b0fd1ce5c70fbe71eea8a9bfb0ab0b1217f1b", "tools/dockerfile/test/csharp_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_arm64@sha256:4d4bc5f15e03f3d3d8fd889670ecde2c66a2e4d2dd9db80733c05c1d90c8a248", diff --git a/tools/docker_runners/examples/concurrent_bazel.sh b/tools/docker_runners/examples/concurrent_bazel.sh new file mode 100755 index 00000000000..e4c667fc197 --- /dev/null +++ b/tools/docker_runners/examples/concurrent_bazel.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This wrapper allows running concurrent instances of bazel build/test by running +# bazel under a docker container. +# This is especially useful for running bazel RBE builds, since inside +# the container, bazel won't have access to local build cache. +# Access to the local workspace is provided by mounting the workspace +# as a volume to the docker container. That also means that any changes +# in the workspace will be visible to the bazel instance running inside +# the container (but also this is a similar scenario to making changes +# to local files when bazel is running normally). + +# Usage: +# tools/docker_runners/examples/concurrent_bazel.sh ANY_NORMAL_BAZEL_FLAGS_HERE + +set -ex + +# change to grpc repo root +cd "$(dirname "$0")/../../.." + +# use the default docker image used for bazel builds +export DOCKERFILE_DIR=tools/dockerfile/test/bazel + +# Bazel RBE uses application default credentials from localhost to authenticate with RBE servers. Use a trick to make the credentails accessible from inside the docker container." +APPLICATION_DEFAULT_CREDENTIALS_DIR="$HOME/.config/gcloud" +export DOCKER_EXTRA_ARGS="-v=${APPLICATION_DEFAULT_CREDENTIALS_DIR}:/application_default_credentials:ro -e=GOOGLE_APPLICATION_CREDENTIALS=/application_default_credentials/application_default_credentials.json" + +# Run bazel inside a docker container (local git workspace will be mounted to the container) +tools/docker_runners/run_in_docker.sh bazel "$@" diff --git a/tools/dockerfile/test/bazel.current_version b/tools/dockerfile/test/bazel.current_version index 93a08864782..ff91869296b 100644 --- a/tools/dockerfile/test/bazel.current_version +++ b/tools/dockerfile/test/bazel.current_version @@ -1 +1 @@ -us-docker.pkg.dev/grpc-testing/testing-images-public/bazel:d64f53cefe4fe5908e00130a7a0dbe6253d9f0b8@sha256:d3291d3129469bd4bdd58b5dd81eb6c5668548a4d6e6c8e952fb372d1874b078 \ No newline at end of file +us-docker.pkg.dev/grpc-testing/testing-images-public/bazel:c8658dbcb51352d5e9c5d9f15282835e45d5ce80@sha256:023fdfd072e278d4129f2c8f8564f6697e767c9847aae8c9411ea853f3faaa6f \ No newline at end of file diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile index ab027114510..5175c12232f 100644 --- a/tools/dockerfile/test/bazel/Dockerfile +++ b/tools/dockerfile/test/bazel/Dockerfile @@ -65,6 +65,17 @@ RUN BAZEL_ARCH_SUFFIX="$(uname -m | sed s/aarch64/arm64/)" \ RUN git config --global --add safe.directory '*' RUN git config --global protocol.file.allow always +#================= +# Setup git to access working directory across docker boundary. +# This avoids the "fatal: detected dubious ownership in repository XYZ" +# git error. Using "--system" makes the setting work +# for all users or even current user is not properly defined +# (which can happen e.g. inside execution environment +# of a bazel action) + +RUN git config --system --add safe.directory '*' +RUN git config --system protocol.file.allow always + RUN apt-get install -y gdb