From 1973b4f9b16443bec6788f85c482c88d99564b1a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 4 Apr 2022 09:47:43 +0200 Subject: [PATCH] Cleanup bazel RBE configs (#29281) * refactor RBE configs * better naming for linux specific RBE configs * update names of RBE configs elsewhere * move partial configs to tools/remote_build/include --- .../linux/grpc_bazel_on_foundry_base.sh | 2 +- tools/remote_build/README.md | 4 +-- .../rbe_base_config.bazelrc} | 30 +++++++++------- .../include/rbe_remote_execution.bazelrc | 35 ++++++++++++++++++ .../include/test_config_common.bazelrc | 24 +++++++++++++ .../{rbe_common.bazelrc => linux.bazelrc} | 29 ++++----------- .../{kokoro.bazelrc => linux_kokoro.bazelrc} | 17 +++------ tools/remote_build/mac.bazelrc | 33 ++++------------- tools/remote_build/windows.bazelrc | 36 ++----------------- 9 files changed, 100 insertions(+), 110 deletions(-) rename tools/remote_build/{manual.bazelrc => include/rbe_base_config.bazelrc} (57%) create mode 100644 tools/remote_build/include/rbe_remote_execution.bazelrc create mode 100644 tools/remote_build/include/test_config_common.bazelrc rename tools/remote_build/{rbe_common.bazelrc => linux.bazelrc} (79%) rename tools/remote_build/{kokoro.bazelrc => linux_kokoro.bazelrc} (59%) diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index 08fdeb5c5c1..d1c7e924899 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -30,7 +30,7 @@ BAZEL_INVOCATION_ID="$(uuidgen)" echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids" tools/bazel \ - --bazelrc=tools/remote_build/kokoro.bazelrc \ + --bazelrc=tools/remote_build/linux_kokoro.bazelrc \ test \ --invocation_id="${BAZEL_INVOCATION_ID}" \ --workspace_status_command=tools/remote_build/workspace_status_kokoro.sh \ diff --git a/tools/remote_build/README.md b/tools/remote_build/README.md index 849ceb4fec3..aa3a780ab08 100644 --- a/tools/remote_build/README.md +++ b/tools/remote_build/README.md @@ -24,13 +24,13 @@ IMPORTANT: The OS from which you run the bazel command needs to always match you For `opt` or `dbg` run this command: ``` # manual run of bazel tests remotely on Foundry -bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=opt //test/... +bazel --bazelrc=tools/remote_build/linux.bazelrc test --config=opt //test/... ``` This also works for sanitizer runs (`asan`, `msan`, `tsan`, `ubsan`): ``` # manual run of bazel tests remotely on Foundry with given sanitizer -bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=asan //test/... +bazel --bazelrc=tools/remote_build/linux.bazelrc test --config=asan //test/... ``` ### Windows diff --git a/tools/remote_build/manual.bazelrc b/tools/remote_build/include/rbe_base_config.bazelrc similarity index 57% rename from tools/remote_build/manual.bazelrc rename to tools/remote_build/include/rbe_base_config.bazelrc index bc20be1ba40..89f1fca181a 100644 --- a/tools/remote_build/manual.bazelrc +++ b/tools/remote_build/include/rbe_base_config.bazelrc @@ -1,4 +1,5 @@ -# Copyright 2018 The gRPC Authors +#@IgnoreInspection BashAddShebang +# Copyright 2022 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,28 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -# bazelrc file for running gRPC tests with Foundry (remote build execution) -# manually from developer's workstation +# bazelrc to configure bazel to use gRPC's main RBE instance for remote cache +# and to upload build results to build event service (for bazel build UI results). +# Note that remote build and test execution is left unconfigured. -import %workspace%/tools/remote_build/rbe_common.bazelrc +startup --host_jvm_args=-Dbazel.DigestFunction=SHA256 -build --remote_cache=grpcs://remotebuildexecution.googleapis.com -build --remote_executor=grpcs://remotebuildexecution.googleapis.com +# the RBE instance to use +build --remote_instance_name=projects/grpc-testing/instances/default_instance -# Enable authentication. Bazel will use application default credentials +# Enable authentication (to be able access the RBE service) +# Bazel will use application default credentials # unless overridden by --google_credentials=service_account_credentials.json -# How to setup credentials: +# How to setup credentials to be able to use bazel RBE locally: # https://cloud.google.com/remote-build-execution/docs/results-ui/getting-started-results-ui build --auth_enabled=true +# use remote cache (remote execution needs to be configured separately) +# Note that remote cache is needed +# not only for build speedup, but also for the test logs +# to become available in ResultStore. +build --remote_cache=grpcs://remotebuildexecution.googleapis.com + # Set flags for uploading to BES in order to view results in the Bazel Build # Results UI. build --bes_backend=grpcs://buildeventservice.googleapis.com build --bes_timeout=600s build --bes_results_url="https://source.cloud.google.com/results/invocations/" build --project_id=grpc-testing - -build --jobs=100 - -# print output for tests that fail (default is "summary") -build --test_output=errors diff --git a/tools/remote_build/include/rbe_remote_execution.bazelrc b/tools/remote_build/include/rbe_remote_execution.bazelrc new file mode 100644 index 00000000000..9436c4a4961 --- /dev/null +++ b/tools/remote_build/include/rbe_remote_execution.bazelrc @@ -0,0 +1,35 @@ +#@IgnoreInspection BashAddShebang +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bazelrc to configure bazel to use RBE remote executors to build and run tests. + +import %workspace%/tools/remote_build/include/rbe_base_config.bazelrc + +# configure backend for remote execution +build --remote_executor=grpcs://remotebuildexecution.googleapis.com + +build --spawn_strategy=remote +build --strategy=Javac=remote +build --strategy=Closure=remote +build --genrule_strategy=remote +build --remote_timeout=7200 # very large value to avoid problems like https://github.com/grpc/grpc/issues/20777 + +# In the remote execution environment, each test gets its own docker containers +# and port server won't be available. +build --define GRPC_PORT_ISOLATED_RUNTIME=1 + +# TODO(jtattermusch): is this still required for remote execution to work? +build --experimental_strict_action_env=true +build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 diff --git a/tools/remote_build/include/test_config_common.bazelrc b/tools/remote_build/include/test_config_common.bazelrc new file mode 100644 index 00000000000..2ee79da7822 --- /dev/null +++ b/tools/remote_build/include/test_config_common.bazelrc @@ -0,0 +1,24 @@ +#@IgnoreInspection BashAddShebang +# Copyright 2022 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bazelrc for common configuration of gRPC tests + +build --verbose_failures=true + +# without verbose gRPC logs the test outputs are not very useful +test --test_env=GRPC_VERBOSITY=debug + +# print output for tests that fail (default is "summary") +build --test_output=errors diff --git a/tools/remote_build/rbe_common.bazelrc b/tools/remote_build/linux.bazelrc similarity index 79% rename from tools/remote_build/rbe_common.bazelrc rename to tools/remote_build/linux.bazelrc index 3d03f4c5812..67e79ca7cb7 100644 --- a/tools/remote_build/rbe_common.bazelrc +++ b/tools/remote_build/linux.bazelrc @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# bazelrc with Foundry setting common to both manual run and runs started by Kokoro -# see https://github.com/bazelbuild/bazel-toolchains/tree/master/bazelrc -# for examples and more documentation +# bazelrc file for running gRPC tests with RBE (on Linux) -startup --host_jvm_args=-Dbazel.DigestFunction=SHA256 +import %workspace%/tools/remote_build/include/rbe_remote_execution.bazelrc +# Next section is linux-specific RBE configuration build --crosstool_top=@rbe_default//cc:toolchain build --extra_toolchains=@rbe_default//config:cc-toolchain # Use custom execution platforms defined in third_party/toolchains @@ -26,29 +25,15 @@ build --extra_execution_platforms=@rbe_default//config:platform build --host_platform=@rbe_default//config:platform build --platforms=@rbe_default//config:platform -build --spawn_strategy=remote -build --strategy=Javac=remote -build --strategy=Closure=remote -build --genrule_strategy=remote -build --remote_timeout=7200 # very large value to avoid problems like https://github.com/grpc/grpc/issues/20777 - -build --remote_instance_name=projects/grpc-testing/instances/default_instance - -build --verbose_failures=true - -build --experimental_strict_action_env=true -build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 - -# don't use port server -build --define GRPC_PORT_ISOLATED_RUNTIME=1 -# without verbose gRPC logs the test outputs are not very useful -test --test_env=GRPC_VERBOSITY=debug - # we assume the default bazel RBE build is on linux, # so filter out stuff that should not be built or run there. build --test_tag_filters=-no_linux build --build_tag_filters=-no_linux +import %workspace%/tools/remote_build/include/test_config_common.bazelrc + +build --jobs=100 + # address sanitizer: most settings are already in %workspace%/.bazelrc # we only need a few additional ones that are Foundry specific build:asan --copt=-gmlt diff --git a/tools/remote_build/kokoro.bazelrc b/tools/remote_build/linux_kokoro.bazelrc similarity index 59% rename from tools/remote_build/kokoro.bazelrc rename to tools/remote_build/linux_kokoro.bazelrc index 5c1b061bce3..8641f3c59da 100644 --- a/tools/remote_build/kokoro.bazelrc +++ b/tools/remote_build/linux_kokoro.bazelrc @@ -12,22 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# bazelrc file for running gRPC tests on Kokoro using Foundry +# bazelrc file for running gRPC tests on Kokoro using RBE (on Linux) +# This file has extra settings for tests that run on CI (kokoro). -import %workspace%/tools/remote_build/rbe_common.bazelrc - -build --remote_cache=grpcs://remotebuildexecution.googleapis.com -build --remote_executor=grpcs://remotebuildexecution.googleapis.com - -build --auth_enabled=true - -build --bes_backend=grpcs://buildeventservice.googleapis.com -build --bes_timeout=600s -build --project_id=grpc-testing +import %workspace%/tools/remote_build/linux.bazelrc # required by kokoro for some reason +# TODO(jtattermusch): is this still actually required? build --test_env=USER=anon build --jobs=200 -build --test_output=errors -build --keep_going=true diff --git a/tools/remote_build/mac.bazelrc b/tools/remote_build/mac.bazelrc index 374782190c4..c39e2446a26 100644 --- a/tools/remote_build/mac.bazelrc +++ b/tools/remote_build/mac.bazelrc @@ -1,16 +1,11 @@ # Configuration for mac bazel test runs on CI # Bazel RBE doesn't currently support MacOS platform, -# so we configure tests to build and run locally, -# but try to use RBE build cache and upload results -# to ResultStore +# so we configure actions to execute locally, +# but we still use RBE for build cache and to upload +# results to ResultStore. -startup --host_jvm_args=-Dbazel.DigestFunction=SHA256 +import %workspace%/tools/remote_build/include/rbe_base_config.bazelrc -# Configure remote cache. Note that remote cache is needed -# not only for build speedup, but also for the test logs -# to become available in ResultStore. -build --remote_cache=grpcs://remotebuildexecution.googleapis.com -build --remote_instance_name=projects/grpc-testing/instances/default_instance # Disable uploading to build cache by default. This is to prevent # polluting the build cache with locally-started builds. # When running on CI, we will override this setting along @@ -18,29 +13,15 @@ build --remote_instance_name=projects/grpc-testing/instances/default_instance # by unintentional cache hits. build --remote_upload_local_results=false -build --auth_enabled=true - -build --verbose_failures=true - build --test_tag_filters=-no_mac build --build_tag_filters=-no_mac -# without verbose gRPC logs the test outputs are not very useful -test --test_env=GRPC_VERBOSITY=debug - -# Set flags for uploading to BES in order to view results in the Bazel Build -# Results UI. -build --bes_backend=grpcs://buildeventservice.googleapis.com -build --bes_timeout=600s -build --bes_results_url="https://source.cloud.google.com/results/invocations/" -build --project_id=grpc-testing - -# print output for tests that fail (default is "summary") -build --test_output=errors - # Enable strict warnings to keep it warning-free. +# TODO(jtattermusch): this option is only used on mac. Is it actually useful? build --define=use_strict_warning=true # Dynamic link cause issues like: `dyld: malformed mach-o: load commands size (59272) > 32768` # https://github.com/bazelbuild/bazel/issues/9190 build --dynamic_mode=off + +import %workspace%/tools/remote_build/include/test_config_common.bazelrc diff --git a/tools/remote_build/windows.bazelrc b/tools/remote_build/windows.bazelrc index c260670f069..ceef31cfed7 100644 --- a/tools/remote_build/windows.bazelrc +++ b/tools/remote_build/windows.bazelrc @@ -1,10 +1,6 @@ -startup --host_jvm_args=-Dbazel.DigestFunction=SHA256 - -build --remote_cache=grpcs://remotebuildexecution.googleapis.com -build --remote_executor=grpcs://remotebuildexecution.googleapis.com - -build --auth_enabled=true +import %workspace%/tools/remote_build/include/rbe_remote_execution.bazelrc +# Next section is windows-specific RBE configuration build --host_crosstool_top=//third_party/toolchains/bazel_0.26.0_rbe_windows:toolchain build --crosstool_top=//third_party/toolchains/bazel_0.26.0_rbe_windows:toolchain build --extra_toolchains=//third_party/toolchains/bazel_0.26.0_rbe_windows:cc-toolchain-x64_windows @@ -16,35 +12,9 @@ build --platforms=//third_party/toolchains:rbe_windows build --shell_executable=C:\\tools\\msys64\\usr\\bin\\bash.exe build --python_path=C:\\Python27\\python.exe -build --spawn_strategy=remote -build --strategy=Javac=remote -build --strategy=Closure=remote -build --genrule_strategy=remote -build --remote_timeout=7200 # very large value to avoid problems like https://github.com/grpc/grpc/issues/20777 - -build --remote_instance_name=projects/grpc-testing/instances/default_instance - -build --verbose_failures=true - -build --experimental_strict_action_env=true -build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 - -# don't use port server -build --define GRPC_PORT_ISOLATED_RUNTIME=1 build --test_tag_filters=-no_windows build --build_tag_filters=-no_windows -# without verbose gRPC logs the test outputs are not very useful -test --test_env=GRPC_VERBOSITY=debug - -# Set flags for uploading to BES in order to view results in the Bazel Build -# Results UI. -build --bes_backend=grpcs://buildeventservice.googleapis.com -build --bes_timeout=600s -build --bes_results_url="https://source.cloud.google.com/results/invocations/" -build --project_id=grpc-testing - build --jobs=100 -# print output for tests that fail (default is "summary") -build --test_output=errors +import %workspace%/tools/remote_build/include/test_config_common.bazelrc