Merge pull request #18280 from nlopezgi/master

Setup rbe_autoconfig for gRPC builds that use RBE

WORKSPACE

@@ -9,13 +9,12 @@ grpc_deps()
 grpc_test_only_deps()
 
 register_execution_platforms(
-    "//third_party/toolchains:rbe_ubuntu1604",
+    "//third_party/toolchains:local",
-    "//third_party/toolchains:rbe_ubuntu1604_large",
+    "//third_party/toolchains:local_large",
     "//third_party/toolchains:rbe_windows",
 )
 
 register_toolchains(
-    "//third_party/toolchains:cc-toolchain-clang-x86_64-default",
     "//third_party/toolchains/bazel_0.23.2_rbe_windows:cc-toolchain-x64_windows",
 )
 
@@ -52,3 +51,24 @@ http_archive(
 load("//bazel:grpc_python_deps.bzl", "grpc_python_deps")
 
 grpc_python_deps()
+
+load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig")
+
+# Create toolchain configuration for remote execution.
+rbe_autoconfig(
+    name = "rbe_default",
+)
+
+load("@bazel_toolchains//rules:environments.bzl", "clang_env")
+load("@bazel_skylib//lib:dicts.bzl", "dicts")
+
+# Create msan toolchain configuration for remote execution.
+rbe_autoconfig(
+    name = "rbe_msan",
+    env = dicts.add(
+        clang_env(),
+        {
+            "BAZEL_LINKOPTS": "-lc++:-lc++abi:-lm",
+        },
+    ),
+)

bazel/grpc_deps.bzl

@@ -173,15 +173,15 @@ def grpc_deps():
             url = "https://github.com/abseil/abseil-cpp/archive/308ce31528a7edfa39f5f6d36142278a0ae1bf45.tar.gz",
         )
 
-    if "com_github_bazelbuild_bazeltoolchains" not in native.existing_rules():
+    if "bazel_toolchains" not in native.existing_rules():
         http_archive(
-            name = "com_github_bazelbuild_bazeltoolchains",
+            name = "bazel_toolchains",
-            strip_prefix = "bazel-toolchains-37419a124bdb9af2fec5b99a973d359b6b899b61",
+            strip_prefix = "bazel-toolchains-cddc376d428ada2927ad359211c3e356bd9c9fbb",
             urls = [
-                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
+                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz",
-                "https://github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
+                "https://github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz",
             ],
-            sha256 = "ee854b5de299138c1f4a2edb5573d22b21d975acfc7aa938f36d30b49ef97498",
+            sha256 = "67335b3563d9b67dc2550b8f27cc689b64fadac491e69ce78763d9ba894cc5cc",
         )
 
     if "bazel_skylib" not in native.existing_rules():

third_party/toolchains/BUILD

@@ -16,18 +16,6 @@ licenses(["notice"])  # Apache v2
 
 package(default_visibility = ["//visibility:public"])
 
-# Latest RBE Ubuntu16_04 container
-# Update every time when a new container is released.
-alias(
-    name = "rbe_ubuntu1604",
-    actual = ":rbe_ubuntu1604_r346485",
-)
-
-alias(
-    name = "rbe_ubuntu1604_large",
-    actual = ":rbe_ubuntu1604_r346485_large",
-)
-
 alias(
     name = "rbe_windows",
     actual = ":rbe_windows_1803",
@@ -60,20 +48,13 @@ platform(
 
 # RBE Ubuntu16_04 r346485
 platform(
-    name = "rbe_ubuntu1604_r346485",
+    name = "rbe_ubuntu1604",
+    parents = ["@rbe_default//config:platform"],
     constraint_values = [
-        "@bazel_tools//platforms:x86_64",
-        "@bazel_tools//platforms:linux",
-        "@bazel_tools//tools/cpp:clang",
-        "@com_github_bazelbuild_bazeltoolchains//constraints:xenial",
-        "@com_github_bazelbuild_bazeltoolchains//constraints/sanitizers:support_msan",
         "//third_party/toolchains/machine_size:standard",
     ],
     remote_execution_properties = """
-        properties: {
+        {PARENT_REMOTE_EXECUTION_PROPERTIES}
-          name: "container-image"
-          value:"docker://gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:f3120a030a19d67626ababdac79cc787e699a1aa924081431285118f87e7b375"
-        }
         properties: {
           name: "gceMachineType"  # Small machines for majority of tests.
           value: "n1-highmem-2"
@@ -97,22 +78,14 @@ platform(
         """,
 )
 
-# RBE Ubuntu16_04 r346485 large
 platform(
-    name = "rbe_ubuntu1604_r346485_large",
+    name = "rbe_ubuntu1604_large",
+    parents = ["@rbe_default//config:platform"],
     constraint_values = [
-        "@bazel_tools//platforms:x86_64",
-        "@bazel_tools//platforms:linux",
-        "@bazel_tools//tools/cpp:clang",
-        "@com_github_bazelbuild_bazeltoolchains//constraints:xenial",
-        "@com_github_bazelbuild_bazeltoolchains//constraints/sanitizers:support_msan",
         "//third_party/toolchains/machine_size:large",
     ],
     remote_execution_properties = """
-        properties: {
+        {PARENT_REMOTE_EXECUTION_PROPERTIES}
-          name: "container-image"
-          value:"docker://gcr.io/cloud-marketplace/google/rbe-ubuntu16-04@sha256:f3120a030a19d67626ababdac79cc787e699a1aa924081431285118f87e7b375"
-        }
         properties: {
           name: "gceMachineType"  # Large machines for some resource demanding tests (TSAN).
           value: "n1-standard-8"
@@ -136,12 +109,18 @@ platform(
     """,
 )
 
-toolchain(
+platform(
-    name = "cc-toolchain-clang-x86_64-default",
+    name = "local",
-    exec_compatible_with = [
+    parents = ["@bazel_tools//platforms:target_platform"],
+    constraint_values = [
+        "//third_party/toolchains/machine_size:standard",
     ],
-    target_compatible_with = [
+)
+
+platform(
+    name = "local_large",
+    parents = ["@bazel_tools//platforms:target_platform"],
+    constraint_values = [
+        "//third_party/toolchains/machine_size:large",
     ],
-    toolchain = "@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.1/bazel_0.20.0/default:cc-compiler-k8",
-    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
 )

tools/remote_build/rbe_common.bazelrc

@@ -19,8 +19,8 @@
 
 startup --host_jvm_args=-Dbazel.DigestFunction=SHA256
 
-build --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.1/bazel_0.20.0/default:toolchain
+build --crosstool_top=@rbe_default//cc:toolchain
-build --extra_toolchains=//third_party/toolchains:cc-toolchain-clang-x86_64-default
+build --extra_toolchains=@rbe_default//config:cc-toolchain
 # Use custom execution platforms defined in third_party/toolchains
 build --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604,//third_party/toolchains:rbe_ubuntu1604_large
 build --host_platform=//third_party/toolchains:rbe_ubuntu1604
@@ -66,9 +66,9 @@ build:msan --cxxopt=--stdlib=libc++
 # setting LD_LIBRARY_PATH is necessary
 # to avoid "libc++.so.1: cannot open shared object file"
 build:msan --action_env=LD_LIBRARY_PATH=/usr/local/lib
-build:msan --host_crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.1/bazel_0.20.0/default:toolchain
+build:msan --host_crosstool_top=@rbe_default//cc:toolchain
 # override the config-agnostic crosstool_top
-build:msan --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.1/bazel_0.20.0/msan:toolchain
+build:msan --crosstool_top=@rbe_msan//cc:toolchain
 
 # thread sanitizer: most settings are already in %workspace%/.bazelrc
 # we only need a few additional ones that are Foundry specific
@@ -84,8 +84,4 @@ build:ubsan --copt=-gmlt
 # TODO(jtattermusch): use more reasonable test timeout
 build:ubsan --test_timeout=3600
 # override the config-agnostic crosstool_top
---crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.1/bazel_0.20.0/ubsan:toolchain
+build:ubsan --crosstool_top=@bazel_toolchains//configs/experimental/ubuntu16_04_clang/1.2/bazel_0.21.0/ubsan:toolchain
-# TODO(jtattermusch): remove this once Foundry adds the env to the docker image.
-# ubsan needs symbolizer to work properly, otherwise the suppression file doesn't work
-# and we get test failures.
-build:ubsan --action_env=UBSAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer

tools/run_tests/sanity/check_bazel_workspace.py

@@ -35,7 +35,7 @@ git_submodule_hashes = {
 }
 
 _BAZEL_SKYLIB_DEP_NAME = 'bazel_skylib'
-_BAZEL_TOOLCHAINS_DEP_NAME = 'com_github_bazelbuild_bazeltoolchains'
+_BAZEL_TOOLCHAINS_DEP_NAME = 'bazel_toolchains'
 _TWISTED_TWISTED_DEP_NAME = 'com_github_twisted_twisted'
 _YAML_PYYAML_DEP_NAME = 'com_github_yaml_pyyaml'
 _TWISTED_INCREMENTAL_DEP_NAME = 'com_github_twisted_incremental'