Merge pull request #24370 from jtattermusch/e2_rbe_pools

RBE: rely on labels to select the right worker pool (instead of hardcoding GCE machine types)

WORKSPACE

@@ -18,12 +18,25 @@ register_toolchains(
     "//third_party/toolchains/bazel_0.26.0_rbe_windows:cc-toolchain-x64_windows",
 )
 
-load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_exec_properties_dict", "custom_exec_properties")
+load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_exec_properties_dict", "custom_exec_properties", "merge_dicts")
 
 custom_exec_properties(
     name = "grpc_custom_exec_properties",
     constants = {
-        "LARGE_MACHINE": create_exec_properties_dict(gce_machine_type = "n1-standard-8"),
+        "LARGE_MACHINE": merge_dicts(
+            create_exec_properties_dict(),
+            # TODO(jtattermusch): specifying 'labels = {"abc": "xyz"}' in create_exec_properties_dict
+            # is not possible without https://github.com/bazelbuild/bazel-toolchains/pull/748
+            # and currently the toolchain we're using is too old for that. To be able to select worker
+            # pools through labels, we use a workaround and populate the corresponding label values
+            # manually (see create_exec_properties_dict logic for how labels get transformed)
+            # Remove this workaround once we transition to a new-enough bazel toolchain.
+            # The next line corresponds to 'labels = {"os": "ubuntu", "machine_size": "large"}'
+            {
+                "label:os": "ubuntu",
+                "label:machine_size": "large",
+            },
+        ),
     },
 )
 
@@ -32,19 +45,24 @@ load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig")
 # Create toolchain configuration for remote execution.
 rbe_autoconfig(
     name = "rbe_default",
-    exec_properties = create_exec_properties_dict(
+    exec_properties = merge_dicts(
+        create_exec_properties_dict(
             docker_add_capabilities = "SYS_PTRACE",
             docker_privileged = True,
-        # n1-highmem-2 is the default (small machine) machine type. Targets
-        # that want to use other machines (such as LARGE_MACHINE) will override
-        # this value.
-        gce_machine_type = "n1-highmem-2",
-        # WARNING: the os_family constraint has only been introduced recently
-        # and older release branches select workers solely based on gce_machine_type.
-        # Worker pools needs to be configured with care to avoid accidentally running
-        # linux jobs on windows pool and vice versa (which would lead to a test breakage)
             os_family = "Linux",
         ),
+        # TODO(jtattermusch): specifying 'labels = {"abc": "xyz"}' in create_exec_properties_dict
+        # is not possible without https://github.com/bazelbuild/bazel-toolchains/pull/748
+        # and currently the toolchain we're using is too old for that. To be able to select worker
+        # pools through labels, we use a workaround and populate the corresponding label values
+        # manually (see create_exec_properties_dict logic for how labels get transformed)
+        # Remove this workaround once we transition to a new-enough bazel toolchain.
+        # The next line corresponds to 'labels = {"os": "ubuntu", "machine_size": "small"}'
+        {
+            "label:os": "ubuntu",
+            "label:machine_size": "small",
+        },
+    ),
     # use exec_properties instead of deprecated remote_execution_properties
     use_legacy_platform_definition = False,
 )

third_party/toolchains/BUILD

@@ -16,7 +16,7 @@ licenses(["notice"])  # Apache v2
 
 package(default_visibility = ["//visibility:public"])
 
-load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_exec_properties_dict")
+load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_exec_properties_dict", "merge_dicts")
 
 alias(
     name = "rbe_windows",
@@ -30,14 +30,19 @@ platform(
         "@bazel_tools//platforms:x86_64",
         "@bazel_tools//platforms:windows",
     ],
-    exec_properties = create_exec_properties_dict(
+    exec_properties = merge_dicts(
+        create_exec_properties_dict(
             # See rbe_win2019/Dockerfile for image details
             container_image = "docker://gcr.io/grpc-testing/rbe_windows2019_withdbg_toolchain@sha256:7b04ee7e29f942adbf4f70edd2ec4ba20a3e7237e1b54f5cae4b239c6ca41105",
-        
-        # Use a different machine type than used on linux to avoid accidentally scheduling linux jobs on windows workers and vice versa on older release branches
-        gce_machine_type = "n1-standard-2",
             os_family = "Windows",
-        # labels only supported starting from https://github.com/bazelbuild/bazel-toolchains/pull/748
-        #labels = {"os": "windows_2019"},
+        ),
+        # TODO(jtattermusch): specifying 'labels = {"abc": "xyz"}' in create_exec_properties_dict
+        # is not possible without https://github.com/bazelbuild/bazel-toolchains/pull/748
+        # and currently the toolchain we're using is too old for that. To be able to select worker
+        # pools through labels, we use a workaround and populate the corresponding label values
+        # manually (see create_exec_properties_dict logic for how labels get transformed)
+        # Remove this workaround once we transition to a new-enough bazel toolchain.
+        # The next line corresponds to 'labels = {"os": "windows_2019", "machine_size": "small"}'
+        {"label:os": "windows_2019", "label:machine_size": "small"}
     ),
 )