diff --git a/tools/bazel.rc b/tools/bazel.rc
index fcbe9337b9f..7348c7488eb 100644
--- a/tools/bazel.rc
+++ b/tools/bazel.rc
@@ -12,6 +12,9 @@ build:opt --copt=-Wframe-larger-than=16384
 
 build:dbg --compilation_mode=dbg
 
+build:windows_opt --compilation_mode=opt
+build:windows_dbg --compilation_mode=dbg
+
 build:asan --strip=never
 build:asan --copt=-fsanitize=address
 build:asan --copt=-O0
diff --git a/tools/internal_ci/windows/grpc_bazel_rbe_dbg.cfg b/tools/internal_ci/windows/grpc_bazel_rbe_dbg.cfg
index 74e5ad61487..b720e10f0db 100644
--- a/tools/internal_ci/windows/grpc_bazel_rbe_dbg.cfg
+++ b/tools/internal_ci/windows/grpc_bazel_rbe_dbg.cfg
@@ -34,8 +34,7 @@ bazel_setting {
 env_vars {
   # flags will be passed to bazel invocation by bazel_rbe.bat
   key: "BAZEL_FLAGS"
-  # TODO(jtattermusch): add --config=dbg once that works
-  value: "--cache_test_results=no"
+  value: "--cache_test_results=no --config=windows_dbg"
 }
 
 env_vars {
diff --git a/tools/internal_ci/windows/grpc_bazel_rbe_opt.cfg b/tools/internal_ci/windows/grpc_bazel_rbe_opt.cfg
index 99b60c4c5ab..7f6bd4f7b3f 100644
--- a/tools/internal_ci/windows/grpc_bazel_rbe_opt.cfg
+++ b/tools/internal_ci/windows/grpc_bazel_rbe_opt.cfg
@@ -34,7 +34,7 @@ bazel_setting {
 env_vars {
   # flags will be passed to bazel invocation by bazel_rbe.bat
   key: "BAZEL_FLAGS"
-  value: "--cache_test_results=no --config=opt"
+  value: "--cache_test_results=no --config=windows_opt"
 }
 env_vars {
   key: "UPLOAD_TEST_RESULTS"
diff --git a/tools/internal_ci/windows/pull_request/grpc_bazel_rbe_dbg.cfg b/tools/internal_ci/windows/pull_request/grpc_bazel_rbe_dbg.cfg
index a682888c7a3..71470d48544 100644
--- a/tools/internal_ci/windows/pull_request/grpc_bazel_rbe_dbg.cfg
+++ b/tools/internal_ci/windows/pull_request/grpc_bazel_rbe_dbg.cfg
@@ -34,6 +34,5 @@ bazel_setting {
 env_vars {
   # flags will be passed to bazel invocation by bazel_rbe.bat
   key: "BAZEL_FLAGS"
-  # TODO(jtattermusch): add --config=dbg once that works
-  value: ""
+  value: " --config=windows_dbg"
 }
diff --git a/tools/internal_ci/windows/pull_request/grpc_bazel_rbe_opt.cfg b/tools/internal_ci/windows/pull_request/grpc_bazel_rbe_opt.cfg
index 027670096fc..120e1496808 100644
--- a/tools/internal_ci/windows/pull_request/grpc_bazel_rbe_opt.cfg
+++ b/tools/internal_ci/windows/pull_request/grpc_bazel_rbe_opt.cfg
@@ -34,5 +34,5 @@ bazel_setting {
 env_vars {
   # flags will be passed to bazel invocation by bazel_rbe.bat
   key: "BAZEL_FLAGS"
-  value: "--config=opt"
+  value: "--config=windows_opt"
 }
diff --git a/tools/remote_build/windows.bazelrc b/tools/remote_build/windows.bazelrc
index e82cdd2c61f..196f99fe68e 100644
--- a/tools/remote_build/windows.bazelrc
+++ b/tools/remote_build/windows.bazelrc
@@ -1,5 +1,9 @@
 startup --host_jvm_args=-Dbazel.DigestFunction=SHA256
 
+# TODO(jtattermusch): Currently building with --compilation_mode=dbg is broken on Windows RBE
+# so we need to explicitly override it to use fastbuild.
+build:windows_dbg --compilation_mode=fastbuild
+
 build --remote_cache=grpcs://remotebuildexecution.googleapis.com
 build --remote_executor=grpcs://remotebuildexecution.googleapis.com